Experiment 2.1: Implement a simple ANN for CIFAR-10 Dataset with the help of TensorFlow and Keras module. Measure the performance and predict some new data

Load data set and print a few data.

import tensorflow as tf
from tensorflow.keras import datasets, layers, models
import numpy as np
import matplotlib.pyplot as plt

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_images, test_images = train_images / 255.0, test_images / 255.0

class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']

plt.figure(figsize=(10, 5))
for i in range(10):
    plt.subplot(1, 10, i + 1)
    plt.xticks([])
    plt.yticks([])
    plt.grid(False)
    plt.imshow(train_images[i])
    plt.xlabel(class_names[train_labels[i][0]])
plt.show()

Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz
170498071/170498071 ━━━━━━━━━━━━━━━━━━━━ 195s 1us/step

Building a model from scartch just to understand the process of forward and back ward pass and training a mode.

import numpy as np
import matplotlib.pyplot as plt

def relu(x):
    return np.maximum(0, x)

def relu_derivative(x):
    return np.where(x > 0, 1, 0)

def softmax(x):
    exps = np.exp(x - np.max(x, axis=1, keepdims=True))
    return exps / np.sum(exps, axis=1, keepdims=True)

def cross_entropy_loss(y_true, y_pred):
    n_samples = y_true.shape[0]
    logp = - np.log(y_pred[range(n_samples), y_true])
    loss = np.sum(logp) / n_samples
    return loss

def cross_entropy_loss_derivative(y_true, y_pred):
    grad = y_pred.copy()
    grad[range(y_true.shape[0]), y_true] -= 1
    return grad / y_true.shape[0]

input_dim = 32 * 32 * 3
hidden_layer1_dim = 128
hidden_layer2_dim = 64
output_dim = 10

np.random.seed(42)
W1 = np.random.randn(input_dim, hidden_layer1_dim) * np.sqrt(2. / input_dim)
b1 = np.zeros((1, hidden_layer1_dim))
W2 = np.random.randn(hidden_layer1_dim, hidden_layer2_dim) * np.sqrt(2. / hidden_layer1_dim)
b2 = np.zeros((1, hidden_layer2_dim))
W3 = np.random.randn(hidden_layer2_dim, output_dim) * np.sqrt(2. / hidden_layer2_dim)
b3 = np.zeros((1, output_dim))

def forward(X):
    z1 = np.dot(X, W1) + b1
    a1 = relu(z1)
    z2 = np.dot(a1, W2) + b2
    a2 = relu(z2)
    z3 = np.dot(a2, W3) + b3
    a3 = softmax(z3)
    return z1, a1, z2, a2, z3, a3

def backward(X, y, z1, a1, z2, a2, z3, a3, learning_rate=0.01):
    global W1, b1, W2, b2, W3, b3

    dz3 = cross_entropy_loss_derivative(y, a3)
    dW3 = np.dot(a2.T, dz3)
    db3 = np.sum(dz3, axis=0, keepdims=True)

    dz2 = np.dot(dz3, W3.T) * relu_derivative(z2)
    dW2 = np.dot(a1.T, dz2)
    db2 = np.sum(dz2, axis=0, keepdims=True)

    dz1 = np.dot(dz2, W2.T) * relu_derivative(z1)
    dW1 = np.dot(X.T, dz1)
    db1 = np.sum(dz1, axis=0, keepdims=True)
    
    W1 -= learning_rate * dW1
    b1 -= learning_rate * db1
    W2 -= learning_rate * dW2
    b2 -= learning_rate * db2
    W3 -= learning_rate * dW3
    b3 -= learning_rate * db3

def load_cifar10():
    import tensorflow as tf
    (train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.cifar10.load_data()
    train_images = train_images.astype('float32') / 255
    test_images = test_images.astype('float32') / 255
    train_labels = train_labels.flatten()
    test_labels = test_labels.flatten()
    return train_images, train_labels, test_images, test_labels

train_images, train_labels, test_images, test_labels = load_cifar10()
train_images_flatten = train_images.reshape(train_images.shape[0], -1)
test_images_flatten = test_images.reshape(test_images.shape[0], -1)

epochs = 30
learning_rate = 0.01
train_accuracy_list = []
val_accuracy_list = []

validation_split = 0.1
num_val_samples = int(validation_split * train_images_flatten.shape[0])
val_images = train_images_flatten[:num_val_samples]
val_labels = train_labels[:num_val_samples]
partial_train_images = train_images_flatten[num_val_samples:]
partial_train_labels = train_labels[num_val_samples:]

for epoch in range(epochs):
    z1, a1, z2, a2, z3, a3 = forward(partial_train_images)
    loss = cross_entropy_loss(partial_train_labels, a3)
    backward(partial_train_images, partial_train_labels, z1, a1, z2, a2, z3, a3, learning_rate)
    train_predictions = np.argmax(a3, axis=1)
    train_accuracy = np.mean(train_predictions == partial_train_labels)
    train_accuracy_list.append(train_accuracy)
    z1_val, a1_val, z2_val, a2_val, z3_val, a3_val = forward(val_images)
    val_predictions = np.argmax(a3_val, axis=1)
    val_accuracy = np.mean(val_predictions == val_labels)
    val_accuracy_list.append(val_accuracy)
    print(f'Epoch {epoch+1}/{epochs}, Train Accuracy: {train_accuracy}, Val Accuracy: {val_accuracy}')

_, _, _, _, _, a3_test = forward(test_images_flatten)
test_predictions = np.argmax(a3_test, axis=1)
test_accuracy = np.mean(test_predictions == test_labels)
print(f'Test accuracy: {test_accuracy}')

Epoch 1/30, Train Accuracy: 0.1128, Val Accuracy: 0.1186
Epoch 2/30, Train Accuracy: 0.11971111111111112, Val Accuracy: 0.128
Epoch 3/30, Train Accuracy: 0.12793333333333334, Val Accuracy: 0.1348
Epoch 4/30, Train Accuracy: 0.13686666666666666, Val Accuracy: 0.1432
Epoch 5/30, Train Accuracy: 0.14486666666666667, Val Accuracy: 0.148
Epoch 6/30, Train Accuracy: 0.14922222222222223, Val Accuracy: 0.152
Epoch 7/30, Train Accuracy: 0.15288888888888888, Val Accuracy: 0.1554
Epoch 8/30, Train Accuracy: 0.15711111111111112, Val Accuracy: 0.1598
Epoch 9/30, Train Accuracy: 0.15982222222222223, Val Accuracy: 0.1632
Epoch 10/30, Train Accuracy: 0.16286666666666666, Val Accuracy: 0.1682
Epoch 11/30, Train Accuracy: 0.16533333333333333, Val Accuracy: 0.1682
Epoch 12/30, Train Accuracy: 0.16857777777777777, Val Accuracy: 0.1718
Epoch 13/30, Train Accuracy: 0.17155555555555554, Val Accuracy: 0.176
Epoch 14/30, Train Accuracy: 0.1749777777777778, Val Accuracy: 0.1806
Epoch 15/30, Train Accuracy: 0.1784888888888889, Val Accuracy: 0.1858
Epoch 16/30, Train Accuracy: 0.18146666666666667, Val Accuracy: 0.1916
Epoch 17/30, Train Accuracy: 0.18457777777777779, Val Accuracy: 0.1952
Epoch 18/30, Train Accuracy: 0.18777777777777777, Val Accuracy: 0.1978
Epoch 19/30, Train Accuracy: 0.19124444444444444, Val Accuracy: 0.201
Epoch 20/30, Train Accuracy: 0.1942, Val Accuracy: 0.2024
Epoch 21/30, Train Accuracy: 0.19686666666666666, Val Accuracy: 0.205
Epoch 22/30, Train Accuracy: 0.19926666666666668, Val Accuracy: 0.2082
Epoch 23/30, Train Accuracy: 0.2021777777777778, Val Accuracy: 0.2092
Epoch 24/30, Train Accuracy: 0.2049777777777778, Val Accuracy: 0.2154
Epoch 25/30, Train Accuracy: 0.20773333333333333, Val Accuracy: 0.218
Epoch 26/30, Train Accuracy: 0.21006666666666668, Val Accuracy: 0.2188
Epoch 27/30, Train Accuracy: 0.21322222222222223, Val Accuracy: 0.2232
Epoch 28/30, Train Accuracy: 0.21588888888888888, Val Accuracy: 0.226
Epoch 29/30, Train Accuracy: 0.21746666666666667, Val Accuracy: 0.2302
Epoch 30/30, Train Accuracy: 0.21988888888888888, Val Accuracy: 0.2298
Test accuracy: 0.2225

plt.figure(figsize=(10, 6))
plt.plot(range(1, epochs + 1), train_accuracy_list, label='Training Accuracy')
plt.plot(range(1, epochs + 1), val_accuracy_list, label='Validation Accuracy')
plt.axhline(y=test_accuracy, color='r', linestyle='--', label='Test Accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.title('Training, Validation, and Test Accuracy over 30 Epochs for custom built model.')
plt.legend()
plt.tight_layout()
plt.show()

Using the tensorflow inbuilt fuction to build and evaluate a model.

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
from sklearn.model_selection import train_test_split

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0
train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

model = models.Sequential([
    layers.Flatten(input_shape=(32, 32, 3)),  
    layers.Dense(128, activation='relu'),  
    layers.Dense(64, activation='relu'),  
    layers.Dense(10, activation='softmax')  
])

model.compile(optimizer='sgd', 
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

history = model.fit(train_images, train_labels, epochs=15,
                    validation_data=(val_images, val_labels))

test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)
print(f'Test accuracy: {test_acc}')

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Epoch 1/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 4s 2ms/step - accuracy: 0.2537 - loss: 2.0486 - val_accuracy: 0.3581 - val_loss: 1.7957
Epoch 2/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.3715 - loss: 1.7650 - val_accuracy: 0.4094 - val_loss: 1.6703
Epoch 3/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.4118 - loss: 1.6684 - val_accuracy: 0.4185 - val_loss: 1.6475
Epoch 4/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.4349 - loss: 1.5989 - val_accuracy: 0.4365 - val_loss: 1.5981
Epoch 5/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.4447 - loss: 1.5610 - val_accuracy: 0.4373 - val_loss: 1.5883
Epoch 6/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.4551 - loss: 1.5266 - val_accuracy: 0.4486 - val_loss: 1.5620
Epoch 7/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.4715 - loss: 1.4915 - val_accuracy: 0.4648 - val_loss: 1.5235
Epoch 8/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.4860 - loss: 1.4676 - val_accuracy: 0.4592 - val_loss: 1.5261
Epoch 9/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.4890 - loss: 1.4468 - val_accuracy: 0.4553 - val_loss: 1.5153
Epoch 10/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.4963 - loss: 1.4325 - val_accuracy: 0.4700 - val_loss: 1.4903
Epoch 11/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5040 - loss: 1.3991 - val_accuracy: 0.4792 - val_loss: 1.4660
Epoch 12/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5119 - loss: 1.3814 - val_accuracy: 0.4740 - val_loss: 1.4770
Epoch 13/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 4s 3ms/step - accuracy: 0.5123 - loss: 1.3727 - val_accuracy: 0.4731 - val_loss: 1.4698
Epoch 14/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5230 - loss: 1.3533 - val_accuracy: 0.4785 - val_loss: 1.4688
Epoch 15/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5269 - loss: 1.3336 - val_accuracy: 0.4853 - val_loss: 1.4508
313/313 - 0s - 2ms/step - accuracy: 0.4948 - loss: 1.4231
Test accuracy: 0.49480000138282776

Measuring the accouracy of the model.

plt.figure(figsize=(10, 6))
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.axhline(y=test_acc, color='r', linestyle='--', label=f'Test Accuracy: {test_acc:.4f}')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.title('Training and Validation Accuracy over 15 Epochs of ANN built using tensorflow model')
plt.legend()
plt.grid(True)
plt.show()

from tensorflow.keras.preprocessing import image
import matplotlib.pyplot as plt

def load_and_preprocess_image(img_path):
    img = image.load_img(img_path, target_size=(32, 32))
    plt.imshow(image.load_img(img_path)) 
    img_array = image.img_to_array(img)  
    img_array = np.expand_dims(img_array, axis=0)  
    img_array = img_array / 255.0  
    return img_array

def predict_image_class(model, img_path):
    new_image = load_and_preprocess_image(img_path)
    predictions = model.predict(new_image)
    predicted_class = np.argmax(predictions)
    class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
    predicted_class_name = class_names[predicted_class]
    return predicted_class_name

new_image_path = 'frog.jpg'
predicted_class_name = predict_image_class(model, new_image_path)
print(f'The model predicts that the image belongs to class: {predicted_class_name}')

1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 71ms/step
The model predicts that the image belongs to class: frog

new_image_path = 'aeroplane.jpg'
predicted_class_name = predict_image_class(model, new_image_path)
print(f'The model predicts that the image belongs to class: {predicted_class_name}')

1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 28ms/step
The model predicts that the image belongs to class: airplane

new_image_path = 'a1.jpg'
predicted_class_name = predict_image_class(model, new_image_path)
print(f'The model predicts that the image belongs to class: {predicted_class_name}')

1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 30ms/step
The model predicts that the image belongs to class: airplane

---

Experiment 2.2: Implement a simple CNN for same data set and compare the performance of the first model and this model

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
from sklearn.model_selection import train_test_split

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

model = models.Sequential([
    layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)),
    layers.MaxPooling2D((2, 2)),
    layers.Conv2D(64, (3, 3), activation='relu'),
    layers.MaxPooling2D((2, 2)),
    layers.Conv2D(64, (3, 3), activation='relu'),
    layers.Flatten(),
    layers.Dense(64, activation='relu'),
    layers.Dense(10, activation='softmax')
])

model.compile(optimizer='sgd',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

history = model.fit(train_images, train_labels, epochs=15, batch_size=64,
                    validation_data=(val_images, val_labels))

test_loss, test_acc1 = model.evaluate(test_images, test_labels, verbose=2)
print(f'Test accuracy: {test_acc1}')

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 10s 14ms/step - accuracy: 0.1452 - loss: 2.2637 - val_accuracy: 0.2823 - val_loss: 1.9922
Epoch 2/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 9s 13ms/step - accuracy: 0.2880 - loss: 1.9626 - val_accuracy: 0.3453 - val_loss: 1.8435
Epoch 3/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 9s 13ms/step - accuracy: 0.3584 - loss: 1.7802 - val_accuracy: 0.3906 - val_loss: 1.6947
Epoch 4/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 10s 14ms/step - accuracy: 0.4121 - loss: 1.6300 - val_accuracy: 0.4212 - val_loss: 1.6565
Epoch 5/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 8s 12ms/step - accuracy: 0.4482 - loss: 1.5340 - val_accuracy: 0.4583 - val_loss: 1.5104
Epoch 6/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 10s 14ms/step - accuracy: 0.4735 - loss: 1.4611 - val_accuracy: 0.4511 - val_loss: 1.5378
Epoch 7/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 10s 15ms/step - accuracy: 0.4952 - loss: 1.4169 - val_accuracy: 0.4760 - val_loss: 1.4040
Epoch 8/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 9s 13ms/step - accuracy: 0.5141 - loss: 1.3605 - val_accuracy: 0.5093 - val_loss: 1.3478
Epoch 9/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 9s 13ms/step - accuracy: 0.5354 - loss: 1.3150 - val_accuracy: 0.5197 - val_loss: 1.3518
Epoch 10/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 10s 15ms/step - accuracy: 0.5503 - loss: 1.2719 - val_accuracy: 0.5527 - val_loss: 1.2477
Epoch 11/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 10s 14ms/step - accuracy: 0.5643 - loss: 1.2325 - val_accuracy: 0.5230 - val_loss: 1.3458
Epoch 12/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 10s 14ms/step - accuracy: 0.5789 - loss: 1.1932 - val_accuracy: 0.5698 - val_loss: 1.2337
Epoch 13/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 12s 17ms/step - accuracy: 0.5930 - loss: 1.1549 - val_accuracy: 0.5899 - val_loss: 1.1699
Epoch 14/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 10s 15ms/step - accuracy: 0.6008 - loss: 1.1321 - val_accuracy: 0.6103 - val_loss: 1.1278
Epoch 15/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 10s 15ms/step - accuracy: 0.6127 - loss: 1.0989 - val_accuracy: 0.6005 - val_loss: 1.1457
313/313 - 1s - 4ms/step - accuracy: 0.5973 - loss: 1.1392
Test accuracy: 0.5972999930381775

plt.figure(figsize=(10, 6))
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.axhline(y=test_acc, c='r',  linestyle='--', label=f'Test Accuracy of ANN: {test_acc:.4f}')
plt.axhline(y=test_acc1, c='g' ,linestyle='--', label=f'Test Accuracy of CNN: {test_acc1:.4f}')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.title('Training and Validation Accuracy over 15 Epochs of CNN built using tensor flow model')
plt.legend()
plt.grid(True)
plt.show()

from tensorflow.keras.preprocessing import image
import matplotlib.pyplot as plt
import numpy as np

# Load and preprocess the new image
def load_and_preprocess_image(img_path):
    plt.imshow(image.load_img(img_path))  # Resize image to 32x32
    img = image.load_img(img_path, target_size=(32, 32))
    img_array = image.img_to_array(img)  # Convert image to array
    img_array = np.expand_dims(img_array, axis=0)  # Add batch dimension
    img_array = img_array / 255.0  # Normalize to [0, 1] range
    return img_array

# Function to predict the class of a new image
def predict_image_class(model, img_path):
    new_image = load_and_preprocess_image(img_path)
    predictions = model.predict(new_image)
    predicted_class = np.argmax(predictions)
    class_names = ['aeroplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
    predicted_class_name = class_names[predicted_class]
    return predicted_class_name

# Example usage
new_image_path = '/content/frog.jpg'
predicted_class_name = predict_image_class(model, new_image_path)
print(f'The model predicts that the image belongs to class: {predicted_class_name}')

1/1 [==============================] - 0s 90ms/step
The model predicts that the image belongs to class: frog

new_image_path = '/content/aeroplane.jpg'
predicted_class_name = predict_image_class(model, new_image_path)
print(f'The model predicts that the image belongs to class: {predicted_class_name}')

1/1 [==============================] - 0s 29ms/step
The model predicts that the image belongs to class: aeroplane

new_image_path = '/content/a1.jpg'
predicted_class_name = predict_image_class(model, new_image_path)
print(f'The model predicts that the image belongs to class: {predicted_class_name}')

1/1 [==============================] - 0s 22ms/step
The model predicts that the image belongs to class: aeroplane

new_image_path = '/content/horse.jpg'
predicted_class_name = predict_image_class(model, new_image_path)
print(f'The model predicts that the image belongs to class: {predicted_class_name}')

1/1 [==============================] - 0s 19ms/step
The model predicts that the image belongs to class: horse

new_image_path = '/content/ship.png'
predicted_class_name = predict_image_class(model, new_image_path)
print(f'The model predicts that the image belongs to class: {predicted_class_name}')

1/1 [==============================] - 0s 41ms/step
The model predicts that the image belongs to class: ship

Experiment 2.3: Understand the behaviour of role of Network Architecture’s complexity. Work on understanding the effect of no of layers, no of neurons in each layer and effect of activation functions on models’ performance.

Dense ANN

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
from sklearn.model_selection import train_test_split

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()
train_images, test_images = train_images / 255.0, test_images / 255.0
train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)


model = models.Sequential([
    layers.Flatten(input_shape=(32, 32, 3)),  
    layers.Dense(256, activation='relu'),  
    layers.Dense(512, activation='relu'),
    layers.Dense(512, activation='relu'),
    layers.Dense(256, activation='relu'),
    layers.Dense(128, activation='relu'),
    layers.Dense(128, activation='relu'),
    layers.Dense(64, activation='relu'),
    layers.Dense(64, activation='relu'),
    layers.Dense(10, activation='softmax')  
])

model.compile(optimizer='sgd', 
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

history = model.fit(train_images, train_labels, epochs=15,
                    validation_data=(val_images, val_labels))

test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)
print(f'Test accuracy: {test_acc}')

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Epoch 1/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 7s 5ms/step - accuracy: 0.1863 - loss: 2.1786 - val_accuracy: 0.2859 - val_loss: 2.0030
Epoch 2/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.3319 - loss: 1.8535 - val_accuracy: 0.3753 - val_loss: 1.7383
Epoch 3/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 5ms/step - accuracy: 0.3816 - loss: 1.7135 - val_accuracy: 0.3890 - val_loss: 1.7013
Epoch 4/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.4151 - loss: 1.6337 - val_accuracy: 0.4313 - val_loss: 1.5896
Epoch 5/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.4343 - loss: 1.5674 - val_accuracy: 0.4232 - val_loss: 1.6246
Epoch 6/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.4513 - loss: 1.5275 - val_accuracy: 0.4389 - val_loss: 1.5740
Epoch 7/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 5ms/step - accuracy: 0.4698 - loss: 1.4774 - val_accuracy: 0.4558 - val_loss: 1.5225
Epoch 8/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.4848 - loss: 1.4395 - val_accuracy: 0.4653 - val_loss: 1.5175
Epoch 9/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 7s 5ms/step - accuracy: 0.5014 - loss: 1.3975 - val_accuracy: 0.4421 - val_loss: 1.5512
Epoch 10/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.5057 - loss: 1.3737 - val_accuracy: 0.4731 - val_loss: 1.5004
Epoch 11/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.5215 - loss: 1.3446 - val_accuracy: 0.4952 - val_loss: 1.4163
Epoch 12/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 5ms/step - accuracy: 0.5268 - loss: 1.3225 - val_accuracy: 0.4925 - val_loss: 1.4296
Epoch 13/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.5433 - loss: 1.2839 - val_accuracy: 0.4983 - val_loss: 1.4384
Epoch 14/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 5ms/step - accuracy: 0.5488 - loss: 1.2590 - val_accuracy: 0.5116 - val_loss: 1.3803
Epoch 15/15
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 5ms/step - accuracy: 0.5580 - loss: 1.2270 - val_accuracy: 0.5087 - val_loss: 1.3851
313/313 - 1s - 2ms/step - accuracy: 0.5087 - loss: 1.3652
Test accuracy: 0.5087000131607056

import matplotlib.pyplot as plt
plt.figure(figsize=(10, 6))
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.axhline(y=test_acc, color='r', linestyle='--', label=f'Test Accuracy: {test_acc:.4f}')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.title('Training and Validation Accuracy over 15 Epochs for dense ANN')
plt.legend()
plt.grid(True)
plt.show()

Dense CNN

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
from sklearn.model_selection import train_test_split

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()
train_images, test_images = train_images / 255.0, test_images / 255.0
train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)


model = models.Sequential([
    layers.Conv2D(256, (3, 3), activation='relu', padding='same', input_shape=(32, 32, 3)),
    layers.MaxPooling2D((2, 2)),
    layers.Conv2D(256, (3, 3), activation='relu', padding='same'),
    layers.MaxPooling2D((2, 2)),
    layers.Conv2D(128, (3, 3), activation='relu', padding='same'),
    layers.Flatten(),
    layers.Dense(128, activation='relu'),
    layers.Dense(128, activation='relu'),
    layers.Dense(64, activation='relu'),
    layers.Dense(64, activation='relu'),
    layers.Dense(10, activation='softmax')
])

model.compile(optimizer='sgd',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

history = model.fit(train_images, train_labels, epochs=15, batch_size=64,
                    validation_data=(val_images, val_labels))

test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)
print(f'Test accuracy: {test_acc}')

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 168s 240ms/step - accuracy: 0.1322 - loss: 2.2889 - val_accuracy: 0.1851 - val_loss: 2.1450
Epoch 2/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 159s 229ms/step - accuracy: 0.2335 - loss: 2.0681 - val_accuracy: 0.3177 - val_loss: 1.8946
Epoch 3/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 165s 238ms/step - accuracy: 0.3290 - loss: 1.8680 - val_accuracy: 0.3894 - val_loss: 1.6980
Epoch 4/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 161s 232ms/step - accuracy: 0.3988 - loss: 1.6747 - val_accuracy: 0.4135 - val_loss: 1.6278
Epoch 5/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 159s 228ms/step - accuracy: 0.4411 - loss: 1.5462 - val_accuracy: 0.4497 - val_loss: 1.4974
Epoch 6/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 158s 228ms/step - accuracy: 0.4740 - loss: 1.4549 - val_accuracy: 0.4725 - val_loss: 1.4256
Epoch 7/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 158s 228ms/step - accuracy: 0.5056 - loss: 1.3737 - val_accuracy: 0.4594 - val_loss: 1.4897
Epoch 8/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 158s 228ms/step - accuracy: 0.5338 - loss: 1.2976 - val_accuracy: 0.5458 - val_loss: 1.2752
Epoch 9/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 159s 229ms/step - accuracy: 0.5583 - loss: 1.2318 - val_accuracy: 0.5233 - val_loss: 1.3216
Epoch 10/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 159s 229ms/step - accuracy: 0.5915 - loss: 1.1573 - val_accuracy: 0.5532 - val_loss: 1.2716
Epoch 11/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 159s 228ms/step - accuracy: 0.6141 - loss: 1.0845 - val_accuracy: 0.5584 - val_loss: 1.2232
Epoch 12/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 160s 230ms/step - accuracy: 0.6383 - loss: 1.0245 - val_accuracy: 0.6020 - val_loss: 1.1348
Epoch 13/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 161s 232ms/step - accuracy: 0.6547 - loss: 0.9722 - val_accuracy: 0.6455 - val_loss: 1.0083
Epoch 14/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 160s 230ms/step - accuracy: 0.6814 - loss: 0.9044 - val_accuracy: 0.5201 - val_loss: 1.4356
Epoch 15/15
695/695 ━━━━━━━━━━━━━━━━━━━━ 159s 229ms/step - accuracy: 0.6992 - loss: 0.8627 - val_accuracy: 0.6263 - val_loss: 1.0840
313/313 - 11s - 34ms/step - accuracy: 0.6286 - loss: 1.0693
Test accuracy: 0.628600001335144

import matplotlib.pyplot as plt
plt.figure(figsize=(10, 6))
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.axhline(y=test_acc, color='r', linestyle='--', label=f'Test Accuracy: {test_acc:.4f}')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.title('Training and Validation Accuracy over 15 Epochs for Dense CNN')
plt.legend()
plt.grid(True)
plt.show()

Activation function experiments

Experiment to study the model's performance when using both Relu and sigmoid activation functions.

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
from sklearn.model_selection import train_test_split

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

model = models.Sequential([
    layers.Flatten(input_shape=(32, 32, 3)), 
    layers.Dense(256, activation='relu'),  
    layers.Dense(512, activation='sigmoid'),
    layers.Dense(512, activation='relu'),
    layers.Dense(256, activation='relu'),
    layers.Dense(128, activation='sigmoid'),
    layers.Dense(128, activation='sigmoid'),
    layers.Dense(64, activation='relu'),
    layers.Dense(64, activation='relu'),
    layers.Dense(10, activation='softmax')  
])

model.compile(optimizer='sgd',  
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])


history = model.fit(train_images, train_labels, epochs=20,
                    validation_data=(val_images, val_labels))

test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)
print(f'Test accuracy: {test_acc}')

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Epoch 1/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 7s 5ms/step - accuracy: 0.0981 - loss: 2.3061 - val_accuracy: 0.1001 - val_loss: 2.3030
Epoch 2/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.1016 - loss: 2.3036 - val_accuracy: 0.0999 - val_loss: 2.3031
Epoch 3/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.1057 - loss: 2.3026 - val_accuracy: 0.1001 - val_loss: 2.3010
Epoch 4/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.1102 - loss: 2.3006 - val_accuracy: 0.1267 - val_loss: 2.2928
Epoch 5/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.1351 - loss: 2.2815 - val_accuracy: 0.1770 - val_loss: 2.1606
Epoch 6/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.1688 - loss: 2.1210 - val_accuracy: 0.1878 - val_loss: 2.0780
Epoch 7/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.1897 - loss: 2.0562 - val_accuracy: 0.1951 - val_loss: 2.0433
Epoch 8/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.1974 - loss: 2.0409 - val_accuracy: 0.2139 - val_loss: 2.0120
Epoch 9/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.2000 - loss: 2.0190 - val_accuracy: 0.2189 - val_loss: 1.9887
Epoch 10/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.2051 - loss: 2.0049 - val_accuracy: 0.2103 - val_loss: 1.9945
Epoch 11/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.2120 - loss: 1.9873 - val_accuracy: 0.2158 - val_loss: 1.9660
Epoch 12/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.2187 - loss: 1.9747 - val_accuracy: 0.2292 - val_loss: 1.9501
Epoch 13/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.2419 - loss: 1.9470 - val_accuracy: 0.2668 - val_loss: 1.8836
Epoch 14/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.2712 - loss: 1.8756 - val_accuracy: 0.2794 - val_loss: 1.8545
Epoch 15/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.2841 - loss: 1.8419 - val_accuracy: 0.3084 - val_loss: 1.8354
Epoch 16/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.2941 - loss: 1.8242 - val_accuracy: 0.3170 - val_loss: 1.8015
Epoch 17/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.3051 - loss: 1.8081 - val_accuracy: 0.3199 - val_loss: 1.7816
Epoch 18/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.3091 - loss: 1.7920 - val_accuracy: 0.3266 - val_loss: 1.7769
Epoch 19/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.3151 - loss: 1.7795 - val_accuracy: 0.3176 - val_loss: 1.7832
Epoch 20/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.3239 - loss: 1.7637 - val_accuracy: 0.3271 - val_loss: 1.7830
313/313 - 1s - 2ms/step - accuracy: 0.3292 - loss: 1.7727
Test accuracy: 0.32919999957084656

import matplotlib.pyplot as plt
plt.figure(figsize=(10, 6))
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.axhline(y=test_acc, color='r', linestyle='--', label=f'Test Accuracy: {test_acc:.4f}')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.title('Training and Validation Accuracy over 20 Epochs with sigmoid and relu activation fucntions.')
plt.legend()
plt.grid(True)
plt.show()

Experiment to study the model's performance when using relu, elu, selu, tanh and sigmiod activation fucntions.

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
from sklearn.model_selection import train_test_split

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()
train_images, test_images = train_images / 255.0, test_images / 255.0
train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

model = models.Sequential([
    layers.Flatten(input_shape=(32, 32, 3)), 
    layers.Dense(128, activation='relu'),  
    layers.Dense(128, activation='tanh'),  
    layers.Dense(128, activation='elu'),   
    layers.Dense(64, activation='selu'),  
    layers.Dense(64, activation='sigmoid'),  
    layers.Dense(32, activation='tanh'),   
    layers.Dense(32, activation='elu'),   
    layers.Dense(10, activation='softmax')  
])

model.compile(optimizer='sgd',  
              loss='sparse_categorical_crossentropy',  
              metrics=['accuracy'])  


history = model.fit(train_images, train_labels, epochs=20,
                    validation_data=(val_images, val_labels))


test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)
print(f'Test accuracy: {test_acc}')

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Epoch 1/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 4s 3ms/step - accuracy: 0.1810 - loss: 2.1876 - val_accuracy: 0.3123 - val_loss: 1.9055
Epoch 2/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 4s 3ms/step - accuracy: 0.3235 - loss: 1.8807 - val_accuracy: 0.3784 - val_loss: 1.7500
Epoch 3/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.3713 - loss: 1.7434 - val_accuracy: 0.4090 - val_loss: 1.6565
Epoch 4/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 4s 3ms/step - accuracy: 0.4003 - loss: 1.6563 - val_accuracy: 0.3953 - val_loss: 1.6640
Epoch 5/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 4s 3ms/step - accuracy: 0.4203 - loss: 1.5955 - val_accuracy: 0.4385 - val_loss: 1.5918
Epoch 6/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 4s 3ms/step - accuracy: 0.4376 - loss: 1.5506 - val_accuracy: 0.4308 - val_loss: 1.5840
Epoch 7/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.4573 - loss: 1.5062 - val_accuracy: 0.4502 - val_loss: 1.5623
Epoch 8/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.4730 - loss: 1.4744 - val_accuracy: 0.4731 - val_loss: 1.4856
Epoch 9/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.4794 - loss: 1.4474 - val_accuracy: 0.4722 - val_loss: 1.4713
Epoch 10/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.4906 - loss: 1.4098 - val_accuracy: 0.4781 - val_loss: 1.4613
Epoch 11/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5026 - loss: 1.3923 - val_accuracy: 0.4848 - val_loss: 1.4514
Epoch 12/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5075 - loss: 1.3779 - val_accuracy: 0.4904 - val_loss: 1.4248
Epoch 13/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5137 - loss: 1.3552 - val_accuracy: 0.4889 - val_loss: 1.4458
Epoch 14/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5159 - loss: 1.3469 - val_accuracy: 0.4911 - val_loss: 1.4230
Epoch 15/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5314 - loss: 1.3091 - val_accuracy: 0.4945 - val_loss: 1.4200
Epoch 16/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5340 - loss: 1.3012 - val_accuracy: 0.4974 - val_loss: 1.4187
Epoch 17/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5407 - loss: 1.2839 - val_accuracy: 0.4988 - val_loss: 1.4023
Epoch 18/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5467 - loss: 1.2719 - val_accuracy: 0.4992 - val_loss: 1.4182
Epoch 19/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 4s 3ms/step - accuracy: 0.5560 - loss: 1.2464 - val_accuracy: 0.4985 - val_loss: 1.4155
Epoch 20/20
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 3s 2ms/step - accuracy: 0.5542 - loss: 1.2455 - val_accuracy: 0.5203 - val_loss: 1.3653
313/313 - 0s - 2ms/step - accuracy: 0.5189 - loss: 1.3473
Test accuracy: 0.5188999772071838

import matplotlib.pyplot as plt
plt.figure(figsize=(10, 6))
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.axhline(y=test_acc, color='r', linestyle='--', label=f'Test Accuracy: {test_acc:.4f}')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.title('Training and Validation Accuracy over 20 Epochs with relu, elu, selu, tanh and sigmiod activation fucntions.')
plt.legend()
plt.grid(True)
plt.show()

Experiment to study the model's performance when using relu, elu, selu, tanh and sigmiod activation fucntions individaully .

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

activation_functions = ['relu', 'tanh', 'elu', 'selu', 'sigmoid']

results = {}

for activation in activation_functions:
    print(f"Training model with {activation} activation...")
    
    model = models.Sequential([
        layers.Flatten(input_shape=(32, 32, 3)),
        layers.Dense(128, activation=activation),
        layers.Dense(128, activation=activation),
        layers.Dense(128, activation=activation),
        layers.Dense(64, activation=activation),
        layers.Dense(64, activation=activation),
        layers.Dense(32, activation=activation),
        layers.Dense(32, activation=activation),
        layers.Dense(10, activation='softmax')
    ])

    model.compile(optimizer='sgd',
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    history = model.fit(train_images, train_labels, epochs=10,
                        validation_data=(val_images, val_labels),
                        verbose=0)
    
    val_acc = max(history.history['val_accuracy'])
    results[activation] = {
        'val_acc': val_acc
    }

    test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)
    results[activation]['test_acc'] = test_acc

    print(f"{activation} - Validation Accuracy: {val_acc:.4f}, Test Accuracy: {test_acc:.4f}")

# Extracting results for plotting
activation_names = []
val_accuracies = []
test_accuracies = []

for activation in activation_functions:
    activation_names.append(activation)
    val_accuracies.append(results[activation]['val_acc'])
    test_accuracies.append(results[activation]['test_acc'])

Training model with relu activation...

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

relu - Validation Accuracy: 0.4625, Test Accuracy: 0.4705
Training model with tanh activation...
tanh - Validation Accuracy: 0.4695, Test Accuracy: 0.4721
Training model with elu activation...
elu - Validation Accuracy: 0.4851, Test Accuracy: 0.4840
Training model with selu activation...
selu - Validation Accuracy: 0.4738, Test Accuracy: 0.4812
Training model with sigmoid activation...
sigmoid - Validation Accuracy: 0.1001, Test Accuracy: 0.1000

plt.figure(figsize=(10, 6))
plt.plot(activation_names, val_accuracies, marker='o', linestyle='-', color='b', label='Validation Accuracy')
plt.plot(activation_names, test_accuracies, marker='x', linestyle='-', color='r', label='Test Accuracy')
plt.title('Max Validation Accuracy vs Test Accuracy for Different Activation Functions')
plt.xlabel('Activation Functions')
plt.ylabel('Accuracy')
plt.xticks(rotation=45)
plt.legend()
plt.grid(True)
plt.tight_layout()
plt.show()

Experiment 2.4: Understand the Epochs and find the optimal number of epochs to train the model by considering Overfitting and Underfitting problems.

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
from sklearn.model_selection import train_test_split

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

model = models.Sequential([
    layers.Flatten(input_shape=(32, 32, 3)),
    layers.Dense(256, activation='relu'),
    layers.Dense(512, activation='relu'),
    layers.Dense(512, activation='relu'),
    layers.Dense(256, activation='relu'),
    layers.Dense(128, activation='relu'),
    layers.Dense(128, activation='relu'),
    layers.Dense(64, activation='relu'),
    layers.Dense(64, activation='relu'),
    layers.Dense(10, activation='softmax')
])

model.compile(optimizer='sgd',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

history = model.fit(train_images, train_labels, epochs=250,
                    validation_data=(val_images, val_labels))

test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)
print(f'Test accuracy: {test_acc}')

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Epoch 1/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 7s 4ms/step - accuracy: 0.2072 - loss: 2.1500 - val_accuracy: 0.3447 - val_loss: 1.8308
Epoch 2/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.3435 - loss: 1.8188 - val_accuracy: 0.3870 - val_loss: 1.7091
Epoch 3/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.3890 - loss: 1.6942 - val_accuracy: 0.3879 - val_loss: 1.7057
Epoch 4/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.4182 - loss: 1.6164 - val_accuracy: 0.4041 - val_loss: 1.6668
Epoch 5/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.4415 - loss: 1.5550 - val_accuracy: 0.4445 - val_loss: 1.5384
Epoch 6/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.4532 - loss: 1.5109 - val_accuracy: 0.4432 - val_loss: 1.5440
Epoch 7/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.4746 - loss: 1.4596 - val_accuracy: 0.4565 - val_loss: 1.5206
Epoch 8/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.4890 - loss: 1.4218 - val_accuracy: 0.4711 - val_loss: 1.4530
Epoch 9/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.5028 - loss: 1.3896 - val_accuracy: 0.4711 - val_loss: 1.4774
Epoch 10/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.5090 - loss: 1.3558 - val_accuracy: 0.4756 - val_loss: 1.4504
Epoch 11/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.5264 - loss: 1.3220 - val_accuracy: 0.4967 - val_loss: 1.4128
Epoch 12/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.5403 - loss: 1.2872 - val_accuracy: 0.4950 - val_loss: 1.4311
Epoch 13/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.5521 - loss: 1.2549 - val_accuracy: 0.5118 - val_loss: 1.3790
Epoch 14/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.5628 - loss: 1.2321 - val_accuracy: 0.5109 - val_loss: 1.3820
Epoch 15/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.5741 - loss: 1.1936 - val_accuracy: 0.4907 - val_loss: 1.4856
Epoch 16/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.5802 - loss: 1.1713 - val_accuracy: 0.5143 - val_loss: 1.4008
Epoch 17/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.5871 - loss: 1.1517 - val_accuracy: 0.4889 - val_loss: 1.4276
Epoch 18/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.6038 - loss: 1.1126 - val_accuracy: 0.5100 - val_loss: 1.3899
Epoch 19/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.6078 - loss: 1.0897 - val_accuracy: 0.5154 - val_loss: 1.4128
Epoch 20/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.6250 - loss: 1.0500 - val_accuracy: 0.5098 - val_loss: 1.4180
Epoch 21/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.6315 - loss: 1.0279 - val_accuracy: 0.4999 - val_loss: 1.4507
Epoch 22/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.6443 - loss: 0.9982 - val_accuracy: 0.5195 - val_loss: 1.3917
Epoch 23/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.6499 - loss: 0.9712 - val_accuracy: 0.5231 - val_loss: 1.4065
Epoch 24/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.6617 - loss: 0.9511 - val_accuracy: 0.5053 - val_loss: 1.4825
Epoch 25/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.6735 - loss: 0.9121 - val_accuracy: 0.5165 - val_loss: 1.4255
Epoch 26/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.6851 - loss: 0.8807 - val_accuracy: 0.4968 - val_loss: 1.4691
Epoch 27/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.6914 - loss: 0.8556 - val_accuracy: 0.5285 - val_loss: 1.4599
Epoch 28/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.7037 - loss: 0.8300 - val_accuracy: 0.5001 - val_loss: 1.5493
Epoch 29/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.7116 - loss: 0.8030 - val_accuracy: 0.4905 - val_loss: 1.6221
Epoch 30/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.7219 - loss: 0.7778 - val_accuracy: 0.5113 - val_loss: 1.5405
Epoch 31/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.7374 - loss: 0.7338 - val_accuracy: 0.5267 - val_loss: 1.5640
Epoch 32/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.7472 - loss: 0.7084 - val_accuracy: 0.5208 - val_loss: 1.5825
Epoch 33/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.7554 - loss: 0.6822 - val_accuracy: 0.5077 - val_loss: 1.7122
Epoch 34/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 7s 5ms/step - accuracy: 0.7661 - loss: 0.6538 - val_accuracy: 0.5228 - val_loss: 1.6666
Epoch 35/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.7726 - loss: 0.6331 - val_accuracy: 0.5215 - val_loss: 1.6935
Epoch 36/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.7867 - loss: 0.5970 - val_accuracy: 0.5122 - val_loss: 1.7492
Epoch 37/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.7955 - loss: 0.5742 - val_accuracy: 0.5194 - val_loss: 1.7451
Epoch 38/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8019 - loss: 0.5460 - val_accuracy: 0.5129 - val_loss: 1.8506
Epoch 39/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8109 - loss: 0.5290 - val_accuracy: 0.5177 - val_loss: 1.7756
Epoch 40/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8186 - loss: 0.5037 - val_accuracy: 0.5163 - val_loss: 1.8981
Epoch 41/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8329 - loss: 0.4698 - val_accuracy: 0.5170 - val_loss: 1.9701
Epoch 42/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8361 - loss: 0.4583 - val_accuracy: 0.5251 - val_loss: 1.9482
Epoch 43/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8475 - loss: 0.4273 - val_accuracy: 0.5136 - val_loss: 1.8617
Epoch 44/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8590 - loss: 0.3978 - val_accuracy: 0.5046 - val_loss: 2.0659
Epoch 45/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8595 - loss: 0.3981 - val_accuracy: 0.5123 - val_loss: 2.0784
Epoch 46/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8609 - loss: 0.3882 - val_accuracy: 0.5176 - val_loss: 2.0736
Epoch 47/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8725 - loss: 0.3618 - val_accuracy: 0.5077 - val_loss: 2.3448
Epoch 48/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8724 - loss: 0.3549 - val_accuracy: 0.5051 - val_loss: 2.1488
Epoch 49/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8805 - loss: 0.3341 - val_accuracy: 0.5258 - val_loss: 2.1849
Epoch 50/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8939 - loss: 0.3080 - val_accuracy: 0.5208 - val_loss: 2.2690
Epoch 51/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8991 - loss: 0.2869 - val_accuracy: 0.5125 - val_loss: 2.2860
Epoch 52/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.8981 - loss: 0.2860 - val_accuracy: 0.5028 - val_loss: 2.5545
Epoch 53/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9038 - loss: 0.2714 - val_accuracy: 0.4974 - val_loss: 2.6274
Epoch 54/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9129 - loss: 0.2462 - val_accuracy: 0.5104 - val_loss: 2.4207
Epoch 55/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 7s 5ms/step - accuracy: 0.9129 - loss: 0.2498 - val_accuracy: 0.5127 - val_loss: 2.4377
Epoch 56/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9164 - loss: 0.2335 - val_accuracy: 0.5046 - val_loss: 2.6103
Epoch 57/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9145 - loss: 0.2349 - val_accuracy: 0.5082 - val_loss: 2.5960
Epoch 58/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9226 - loss: 0.2174 - val_accuracy: 0.5176 - val_loss: 2.6766
Epoch 59/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9247 - loss: 0.2153 - val_accuracy: 0.5221 - val_loss: 2.6882
Epoch 60/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9265 - loss: 0.2095 - val_accuracy: 0.5032 - val_loss: 2.6744
Epoch 61/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9303 - loss: 0.2003 - val_accuracy: 0.5152 - val_loss: 2.7777
Epoch 62/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9336 - loss: 0.1936 - val_accuracy: 0.5051 - val_loss: 2.7956
Epoch 63/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9340 - loss: 0.1918 - val_accuracy: 0.5176 - val_loss: 2.7753
Epoch 64/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9413 - loss: 0.1681 - val_accuracy: 0.5156 - val_loss: 2.7014
Epoch 65/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9495 - loss: 0.1467 - val_accuracy: 0.5073 - val_loss: 2.8085
Epoch 66/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9416 - loss: 0.1669 - val_accuracy: 0.5145 - val_loss: 2.8467
Epoch 67/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9468 - loss: 0.1527 - val_accuracy: 0.5179 - val_loss: 2.7407
Epoch 68/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9598 - loss: 0.1209 - val_accuracy: 0.5129 - val_loss: 3.0803
Epoch 69/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9463 - loss: 0.1543 - val_accuracy: 0.5059 - val_loss: 2.9726
Epoch 70/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9534 - loss: 0.1369 - val_accuracy: 0.5210 - val_loss: 3.1842
Epoch 71/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 10s 4ms/step - accuracy: 0.9452 - loss: 0.1587 - val_accuracy: 0.5105 - val_loss: 3.0775
Epoch 72/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9519 - loss: 0.1346 - val_accuracy: 0.5158 - val_loss: 3.0867
Epoch 73/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9624 - loss: 0.1061 - val_accuracy: 0.5226 - val_loss: 3.1048
Epoch 74/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9576 - loss: 0.1248 - val_accuracy: 0.5152 - val_loss: 3.1157
Epoch 75/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9575 - loss: 0.1264 - val_accuracy: 0.5021 - val_loss: 3.2385
Epoch 76/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9599 - loss: 0.1168 - val_accuracy: 0.5212 - val_loss: 3.0493
Epoch 77/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9741 - loss: 0.0765 - val_accuracy: 0.5172 - val_loss: 3.1752
Epoch 78/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9672 - loss: 0.1003 - val_accuracy: 0.5197 - val_loss: 3.1684
Epoch 79/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9703 - loss: 0.0893 - val_accuracy: 0.5156 - val_loss: 3.2200
Epoch 80/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9685 - loss: 0.0921 - val_accuracy: 0.5140 - val_loss: 3.2555
Epoch 81/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9719 - loss: 0.0858 - val_accuracy: 0.5077 - val_loss: 3.2600
Epoch 82/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9743 - loss: 0.0756 - val_accuracy: 0.5023 - val_loss: 3.2612
Epoch 83/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9672 - loss: 0.0959 - val_accuracy: 0.5210 - val_loss: 3.0783
Epoch 84/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9753 - loss: 0.0756 - val_accuracy: 0.5176 - val_loss: 3.5292
Epoch 85/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9680 - loss: 0.0917 - val_accuracy: 0.5167 - val_loss: 3.5173
Epoch 86/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9748 - loss: 0.0765 - val_accuracy: 0.5154 - val_loss: 3.4571
Epoch 87/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9715 - loss: 0.0819 - val_accuracy: 0.5249 - val_loss: 3.4263
Epoch 88/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9677 - loss: 0.0940 - val_accuracy: 0.5017 - val_loss: 3.7582
Epoch 89/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9814 - loss: 0.0573 - val_accuracy: 0.5289 - val_loss: 3.3295
Epoch 90/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9836 - loss: 0.0496 - val_accuracy: 0.5197 - val_loss: 3.3825
Epoch 91/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9777 - loss: 0.0664 - val_accuracy: 0.5266 - val_loss: 3.6913
Epoch 92/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9815 - loss: 0.0564 - val_accuracy: 0.5186 - val_loss: 3.6086
Epoch 93/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9719 - loss: 0.0829 - val_accuracy: 0.4844 - val_loss: 3.6415
Epoch 94/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9794 - loss: 0.0624 - val_accuracy: 0.5190 - val_loss: 3.5670
Epoch 95/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9765 - loss: 0.0694 - val_accuracy: 0.5269 - val_loss: 3.4937
Epoch 96/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9831 - loss: 0.0520 - val_accuracy: 0.5161 - val_loss: 3.5786
Epoch 97/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9801 - loss: 0.0617 - val_accuracy: 0.5312 - val_loss: 3.4056
Epoch 98/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9878 - loss: 0.0381 - val_accuracy: 0.5161 - val_loss: 3.7531
Epoch 99/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9778 - loss: 0.0668 - val_accuracy: 0.5248 - val_loss: 3.6333
Epoch 100/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9931 - loss: 0.0210 - val_accuracy: 0.5188 - val_loss: 4.2104
Epoch 101/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9787 - loss: 0.0622 - val_accuracy: 0.5244 - val_loss: 3.6484
Epoch 102/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9782 - loss: 0.0626 - val_accuracy: 0.5163 - val_loss: 3.6079
Epoch 103/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9873 - loss: 0.0393 - val_accuracy: 0.5141 - val_loss: 3.8679
Epoch 104/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9809 - loss: 0.0592 - val_accuracy: 0.5158 - val_loss: 3.6717
Epoch 105/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9860 - loss: 0.0421 - val_accuracy: 0.5237 - val_loss: 3.9065
Epoch 106/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9771 - loss: 0.0664 - val_accuracy: 0.5233 - val_loss: 3.7700
Epoch 107/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9903 - loss: 0.0296 - val_accuracy: 0.5082 - val_loss: 3.8700
Epoch 108/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9785 - loss: 0.0656 - val_accuracy: 0.5143 - val_loss: 3.6294
Epoch 109/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9888 - loss: 0.0355 - val_accuracy: 0.5334 - val_loss: 3.6645
Epoch 110/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9885 - loss: 0.0375 - val_accuracy: 0.5264 - val_loss: 3.9774
Epoch 111/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9903 - loss: 0.0301 - val_accuracy: 0.5287 - val_loss: 3.8663
Epoch 112/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9812 - loss: 0.0601 - val_accuracy: 0.5185 - val_loss: 3.8645
Epoch 113/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9886 - loss: 0.0357 - val_accuracy: 0.5069 - val_loss: 3.7365
Epoch 114/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9787 - loss: 0.0632 - val_accuracy: 0.5316 - val_loss: 3.8934
Epoch 115/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9837 - loss: 0.0497 - val_accuracy: 0.5174 - val_loss: 3.8121
Epoch 116/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9857 - loss: 0.0425 - val_accuracy: 0.5273 - val_loss: 3.9523
Epoch 117/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9934 - loss: 0.0206 - val_accuracy: 0.5188 - val_loss: 3.7524
Epoch 118/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9879 - loss: 0.0356 - val_accuracy: 0.5285 - val_loss: 3.8977
Epoch 119/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9879 - loss: 0.0368 - val_accuracy: 0.5204 - val_loss: 4.0439
Epoch 120/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9954 - loss: 0.0139 - val_accuracy: 0.5221 - val_loss: 4.1767
Epoch 121/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9786 - loss: 0.0640 - val_accuracy: 0.5192 - val_loss: 4.0237
Epoch 122/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9859 - loss: 0.0413 - val_accuracy: 0.5228 - val_loss: 4.0423
Epoch 123/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9918 - loss: 0.0232 - val_accuracy: 0.5291 - val_loss: 4.3082
Epoch 124/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9838 - loss: 0.0486 - val_accuracy: 0.5260 - val_loss: 3.8466
Epoch 125/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9931 - loss: 0.0227 - val_accuracy: 0.5177 - val_loss: 4.0705
Epoch 126/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9876 - loss: 0.0381 - val_accuracy: 0.5289 - val_loss: 4.0207
Epoch 127/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9862 - loss: 0.0413 - val_accuracy: 0.5219 - val_loss: 3.7121
Epoch 128/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9912 - loss: 0.0249 - val_accuracy: 0.5296 - val_loss: 4.1307
Epoch 129/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9947 - loss: 0.0179 - val_accuracy: 0.5134 - val_loss: 4.1618
Epoch 130/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9780 - loss: 0.0638 - val_accuracy: 0.5235 - val_loss: 4.2559
Epoch 131/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9945 - loss: 0.0184 - val_accuracy: 0.5311 - val_loss: 4.2351
Epoch 132/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9954 - loss: 0.0140 - val_accuracy: 0.5384 - val_loss: 4.3834
Epoch 133/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9998 - loss: 0.0012 - val_accuracy: 0.5419 - val_loss: 4.4810
Epoch 134/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9998 - loss: 8.6617e-04 - val_accuracy: 0.5408 - val_loss: 4.5456
Epoch 135/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 3.5140e-04 - val_accuracy: 0.5438 - val_loss: 4.5911
Epoch 136/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.0309e-04 - val_accuracy: 0.5426 - val_loss: 4.6324
Epoch 137/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 0.9999 - loss: 2.8975e-04 - val_accuracy: 0.5460 - val_loss: 4.6782
Epoch 138/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.1135e-04 - val_accuracy: 0.5467 - val_loss: 4.7128
Epoch 139/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 9.9011e-05 - val_accuracy: 0.5465 - val_loss: 4.7379
Epoch 140/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 8.6355e-05 - val_accuracy: 0.5467 - val_loss: 4.7666
Epoch 141/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 8.8201e-05 - val_accuracy: 0.5467 - val_loss: 4.7892
Epoch 142/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 7.3280e-05 - val_accuracy: 0.5469 - val_loss: 4.8145
Epoch 143/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 6.4985e-05 - val_accuracy: 0.5476 - val_loss: 4.8357
Epoch 144/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 6.2649e-05 - val_accuracy: 0.5476 - val_loss: 4.8549
Epoch 145/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 6.3361e-05 - val_accuracy: 0.5467 - val_loss: 4.8764
Epoch 146/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 5.9051e-05 - val_accuracy: 0.5473 - val_loss: 4.8938
Epoch 147/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 5.4666e-05 - val_accuracy: 0.5474 - val_loss: 4.9101
Epoch 148/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 5.5628e-05 - val_accuracy: 0.5476 - val_loss: 4.9278
Epoch 149/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 4.8171e-05 - val_accuracy: 0.5476 - val_loss: 4.9431
Epoch 150/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 4.5762e-05 - val_accuracy: 0.5478 - val_loss: 4.9590
Epoch 151/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 4.4647e-05 - val_accuracy: 0.5483 - val_loss: 4.9737
Epoch 152/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 4.5413e-05 - val_accuracy: 0.5482 - val_loss: 4.9877
Epoch 153/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 4.2522e-05 - val_accuracy: 0.5487 - val_loss: 5.0019
Epoch 154/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 3.9756e-05 - val_accuracy: 0.5491 - val_loss: 5.0152
Epoch 155/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 4.0016e-05 - val_accuracy: 0.5487 - val_loss: 5.0269
Epoch 156/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 3.6800e-05 - val_accuracy: 0.5485 - val_loss: 5.0400
Epoch 157/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 3.5591e-05 - val_accuracy: 0.5487 - val_loss: 5.0517
Epoch 158/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 3.5389e-05 - val_accuracy: 0.5489 - val_loss: 5.0628
Epoch 159/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 3.4926e-05 - val_accuracy: 0.5498 - val_loss: 5.0736
Epoch 160/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 3.4409e-05 - val_accuracy: 0.5491 - val_loss: 5.0851
Epoch 161/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 3.3260e-05 - val_accuracy: 0.5498 - val_loss: 5.0958
Epoch 162/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 3.3408e-05 - val_accuracy: 0.5492 - val_loss: 5.1062
Epoch 163/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.9798e-05 - val_accuracy: 0.5496 - val_loss: 5.1154
Epoch 164/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 5ms/step - accuracy: 1.0000 - loss: 2.7216e-05 - val_accuracy: 0.5491 - val_loss: 5.1256
Epoch 165/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.8598e-05 - val_accuracy: 0.5489 - val_loss: 5.1350
Epoch 166/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.7904e-05 - val_accuracy: 0.5489 - val_loss: 5.1449
Epoch 167/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.5934e-05 - val_accuracy: 0.5489 - val_loss: 5.1533
Epoch 168/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.7172e-05 - val_accuracy: 0.5489 - val_loss: 5.1621
Epoch 169/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.5764e-05 - val_accuracy: 0.5494 - val_loss: 5.1712
Epoch 170/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.5062e-05 - val_accuracy: 0.5494 - val_loss: 5.1799
Epoch 171/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.4111e-05 - val_accuracy: 0.5494 - val_loss: 5.1880
Epoch 172/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.4673e-05 - val_accuracy: 0.5494 - val_loss: 5.1967
Epoch 173/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.3050e-05 - val_accuracy: 0.5489 - val_loss: 5.2047
Epoch 174/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.3125e-05 - val_accuracy: 0.5494 - val_loss: 5.2128
Epoch 175/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.2368e-05 - val_accuracy: 0.5494 - val_loss: 5.2200
Epoch 176/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.3197e-05 - val_accuracy: 0.5496 - val_loss: 5.2273
Epoch 177/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.4221e-05 - val_accuracy: 0.5494 - val_loss: 5.2351
Epoch 178/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.3069e-05 - val_accuracy: 0.5496 - val_loss: 5.2424
Epoch 179/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.1643e-05 - val_accuracy: 0.5496 - val_loss: 5.2496
Epoch 180/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.1028e-05 - val_accuracy: 0.5498 - val_loss: 5.2565
Epoch 181/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.0361e-05 - val_accuracy: 0.5501 - val_loss: 5.2636
Epoch 182/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.0457e-05 - val_accuracy: 0.5500 - val_loss: 5.2704
Epoch 183/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.9859e-05 - val_accuracy: 0.5503 - val_loss: 5.2771
Epoch 184/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 2.0752e-05 - val_accuracy: 0.5505 - val_loss: 5.2840
Epoch 185/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.9516e-05 - val_accuracy: 0.5507 - val_loss: 5.2902
Epoch 186/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.9812e-05 - val_accuracy: 0.5503 - val_loss: 5.2966
Epoch 187/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.9540e-05 - val_accuracy: 0.5507 - val_loss: 5.3028
Epoch 188/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.9347e-05 - val_accuracy: 0.5503 - val_loss: 5.3093
Epoch 189/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.8699e-05 - val_accuracy: 0.5503 - val_loss: 5.3152
Epoch 190/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.8838e-05 - val_accuracy: 0.5501 - val_loss: 5.3210
Epoch 191/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.7265e-05 - val_accuracy: 0.5505 - val_loss: 5.3272
Epoch 192/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.7352e-05 - val_accuracy: 0.5505 - val_loss: 5.3330
Epoch 193/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.6931e-05 - val_accuracy: 0.5505 - val_loss: 5.3389
Epoch 194/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.6377e-05 - val_accuracy: 0.5505 - val_loss: 5.3447
Epoch 195/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.6218e-05 - val_accuracy: 0.5505 - val_loss: 5.3500
Epoch 196/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.5794e-05 - val_accuracy: 0.5507 - val_loss: 5.3555
Epoch 197/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.5729e-05 - val_accuracy: 0.5507 - val_loss: 5.3610
Epoch 198/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.5463e-05 - val_accuracy: 0.5505 - val_loss: 5.3662
Epoch 199/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.5675e-05 - val_accuracy: 0.5505 - val_loss: 5.3717
Epoch 200/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.5103e-05 - val_accuracy: 0.5505 - val_loss: 5.3769
Epoch 201/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.4643e-05 - val_accuracy: 0.5505 - val_loss: 5.3819
Epoch 202/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.4436e-05 - val_accuracy: 0.5503 - val_loss: 5.3872
Epoch 203/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.4275e-05 - val_accuracy: 0.5501 - val_loss: 5.3923
Epoch 204/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.4489e-05 - val_accuracy: 0.5509 - val_loss: 5.3972
Epoch 205/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.4372e-05 - val_accuracy: 0.5507 - val_loss: 5.4020
Epoch 206/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.4198e-05 - val_accuracy: 0.5507 - val_loss: 5.4072
Epoch 207/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.3842e-05 - val_accuracy: 0.5509 - val_loss: 5.4121
Epoch 208/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.3949e-05 - val_accuracy: 0.5509 - val_loss: 5.4169
Epoch 209/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.3284e-05 - val_accuracy: 0.5505 - val_loss: 5.4216
Epoch 210/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.4151e-05 - val_accuracy: 0.5507 - val_loss: 5.4264
Epoch 211/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.3302e-05 - val_accuracy: 0.5507 - val_loss: 5.4312
Epoch 212/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.3476e-05 - val_accuracy: 0.5509 - val_loss: 5.4354
Epoch 213/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.3220e-05 - val_accuracy: 0.5509 - val_loss: 5.4398
Epoch 214/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.2843e-05 - val_accuracy: 0.5509 - val_loss: 5.4445
Epoch 215/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.2868e-05 - val_accuracy: 0.5509 - val_loss: 5.4492
Epoch 216/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.2513e-05 - val_accuracy: 0.5509 - val_loss: 5.4533
Epoch 217/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.2332e-05 - val_accuracy: 0.5509 - val_loss: 5.4577
Epoch 218/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.2085e-05 - val_accuracy: 0.5507 - val_loss: 5.4621
Epoch 219/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.2958e-05 - val_accuracy: 0.5507 - val_loss: 5.4662
Epoch 220/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.2623e-05 - val_accuracy: 0.5507 - val_loss: 5.4705
Epoch 221/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.2175e-05 - val_accuracy: 0.5509 - val_loss: 5.4746
Epoch 222/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.1611e-05 - val_accuracy: 0.5509 - val_loss: 5.4788
Epoch 223/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.1531e-05 - val_accuracy: 0.5509 - val_loss: 5.4829
Epoch 224/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.1761e-05 - val_accuracy: 0.5510 - val_loss: 5.4869
Epoch 225/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.1382e-05 - val_accuracy: 0.5509 - val_loss: 5.4910
Epoch 226/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.0783e-05 - val_accuracy: 0.5509 - val_loss: 5.4949
Epoch 227/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.1820e-05 - val_accuracy: 0.5509 - val_loss: 5.4989
Epoch 228/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.0910e-05 - val_accuracy: 0.5507 - val_loss: 5.5029
Epoch 229/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.0919e-05 - val_accuracy: 0.5505 - val_loss: 5.5068
Epoch 230/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.0761e-05 - val_accuracy: 0.5505 - val_loss: 5.5108
Epoch 231/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.0857e-05 - val_accuracy: 0.5507 - val_loss: 5.5147
Epoch 232/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.1156e-05 - val_accuracy: 0.5507 - val_loss: 5.5183
Epoch 233/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.0924e-05 - val_accuracy: 0.5507 - val_loss: 5.5221
Epoch 234/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 7s 5ms/step - accuracy: 1.0000 - loss: 1.0772e-05 - val_accuracy: 0.5505 - val_loss: 5.5260
Epoch 235/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 5ms/step - accuracy: 1.0000 - loss: 1.0601e-05 - val_accuracy: 0.5505 - val_loss: 5.5298
Epoch 236/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 9.8699e-06 - val_accuracy: 0.5505 - val_loss: 5.5334
Epoch 237/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 9.8145e-06 - val_accuracy: 0.5503 - val_loss: 5.5371
Epoch 238/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.0133e-05 - val_accuracy: 0.5505 - val_loss: 5.5406
Epoch 239/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.0168e-05 - val_accuracy: 0.5503 - val_loss: 5.5444
Epoch 240/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 9.9897e-06 - val_accuracy: 0.5503 - val_loss: 5.5479
Epoch 241/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.0291e-05 - val_accuracy: 0.5503 - val_loss: 5.5514
Epoch 242/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 1.0225e-05 - val_accuracy: 0.5505 - val_loss: 5.5549
Epoch 243/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 9.8761e-06 - val_accuracy: 0.5505 - val_loss: 5.5583
Epoch 244/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 9.8813e-06 - val_accuracy: 0.5507 - val_loss: 5.5617
Epoch 245/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 9.7264e-06 - val_accuracy: 0.5507 - val_loss: 5.5653
Epoch 246/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 9.2280e-06 - val_accuracy: 0.5507 - val_loss: 5.5686
Epoch 247/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 9.2034e-06 - val_accuracy: 0.5505 - val_loss: 5.5720
Epoch 248/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 9.2836e-06 - val_accuracy: 0.5505 - val_loss: 5.5752
Epoch 249/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 7s 5ms/step - accuracy: 1.0000 - loss: 9.4271e-06 - val_accuracy: 0.5505 - val_loss: 5.5785
Epoch 250/250
1389/1389 ━━━━━━━━━━━━━━━━━━━━ 6s 4ms/step - accuracy: 1.0000 - loss: 9.5810e-06 - val_accuracy: 0.5507 - val_loss: 5.5819
313/313 - 1s - 2ms/step - accuracy: 0.5437 - loss: 5.5596
Test accuracy: 0.5436999797821045

import matplotlib.pyplot as plt
plt.figure(figsize=(10, 6))
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.axhline(y=test_acc, color='r', linestyle='--', label=f'Test Accuracy: {test_acc:.4f}')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.title('Training and Validation Accuracy over Epochs')
plt.legend()
plt.grid(True)
plt.show()

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
from sklearn.model_selection import train_test_split

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

model = models.Sequential([
    layers.Conv2D(256, (3, 3), activation='relu', padding='same', input_shape=(32, 32, 3)),
    layers.MaxPooling2D((2, 2)),
    layers.Conv2D(256, (3, 3), activation='relu', padding='same'),
    layers.MaxPooling2D((2, 2)),
    layers.Conv2D(128, (3, 3), activation='relu', padding='same'),
    layers.Flatten(),
    layers.Dense(128, activation='relu'),
    layers.Dense(128, activation='relu'),
    layers.Dense(64, activation='relu'),
    layers.Dense(64, activation='relu'),
    layers.Dense(10, activation='softmax')
])

model.compile(optimizer='sgd',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

history = model.fit(train_images, train_labels, epochs=150, batch_size=64,
                    validation_data=(val_images, val_labels))

test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)
print(f'Test accuracy: {test_acc}')

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 213ms/step - accuracy: 0.1529 - loss: 2.2909 - val_accuracy: 0.2049 - val_loss: 2.1162
Epoch 2/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 147s 211ms/step - accuracy: 0.2338 - loss: 2.0732 - val_accuracy: 0.2547 - val_loss: 2.0559
Epoch 3/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 212ms/step - accuracy: 0.3175 - loss: 1.8878 - val_accuracy: 0.3784 - val_loss: 1.7186
Epoch 4/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 213ms/step - accuracy: 0.3914 - loss: 1.6989 - val_accuracy: 0.4283 - val_loss: 1.5894
Epoch 5/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 152s 218ms/step - accuracy: 0.4329 - loss: 1.5657 - val_accuracy: 0.4652 - val_loss: 1.4914
Epoch 6/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 215ms/step - accuracy: 0.4649 - loss: 1.4826 - val_accuracy: 0.4767 - val_loss: 1.4150
Epoch 7/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 147s 212ms/step - accuracy: 0.4981 - loss: 1.3985 - val_accuracy: 0.4842 - val_loss: 1.4140
Epoch 8/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 145s 208ms/step - accuracy: 0.5295 - loss: 1.3111 - val_accuracy: 0.5404 - val_loss: 1.2946
Epoch 9/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 145s 208ms/step - accuracy: 0.5601 - loss: 1.2292 - val_accuracy: 0.5512 - val_loss: 1.2667
Epoch 10/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 146s 211ms/step - accuracy: 0.5833 - loss: 1.1665 - val_accuracy: 0.5935 - val_loss: 1.1524
Epoch 11/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 146s 210ms/step - accuracy: 0.6047 - loss: 1.1109 - val_accuracy: 0.6256 - val_loss: 1.0840
Epoch 12/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 145s 208ms/step - accuracy: 0.6248 - loss: 1.0467 - val_accuracy: 0.5257 - val_loss: 1.3076
Epoch 13/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 145s 209ms/step - accuracy: 0.6493 - loss: 0.9862 - val_accuracy: 0.5842 - val_loss: 1.1950
Epoch 14/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 145s 209ms/step - accuracy: 0.6695 - loss: 0.9366 - val_accuracy: 0.6425 - val_loss: 1.0333
Epoch 15/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 145s 209ms/step - accuracy: 0.6937 - loss: 0.8733 - val_accuracy: 0.6385 - val_loss: 1.0369
Epoch 16/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 146s 209ms/step - accuracy: 0.7082 - loss: 0.8292 - val_accuracy: 0.6162 - val_loss: 1.1063
Epoch 17/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 146s 210ms/step - accuracy: 0.7294 - loss: 0.7724 - val_accuracy: 0.6443 - val_loss: 1.0450
Epoch 18/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 145s 209ms/step - accuracy: 0.7465 - loss: 0.7207 - val_accuracy: 0.6749 - val_loss: 0.9534
Epoch 19/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 145s 209ms/step - accuracy: 0.7604 - loss: 0.6780 - val_accuracy: 0.6850 - val_loss: 0.9292
Epoch 20/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 145s 209ms/step - accuracy: 0.7819 - loss: 0.6247 - val_accuracy: 0.7026 - val_loss: 0.8973
Epoch 21/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 0.7965 - loss: 0.5770 - val_accuracy: 0.6992 - val_loss: 0.9112
Epoch 22/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 146s 210ms/step - accuracy: 0.8140 - loss: 0.5369 - val_accuracy: 0.6621 - val_loss: 1.0261
Epoch 23/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 146s 210ms/step - accuracy: 0.8253 - loss: 0.4980 - val_accuracy: 0.7078 - val_loss: 0.8947
Epoch 24/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 146s 210ms/step - accuracy: 0.8488 - loss: 0.4397 - val_accuracy: 0.6884 - val_loss: 0.9690
Epoch 25/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 146s 210ms/step - accuracy: 0.8618 - loss: 0.3996 - val_accuracy: 0.6443 - val_loss: 1.2007
Epoch 26/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 146s 210ms/step - accuracy: 0.8777 - loss: 0.3512 - val_accuracy: 0.6835 - val_loss: 1.0782
Epoch 27/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 146s 210ms/step - accuracy: 0.8953 - loss: 0.3093 - val_accuracy: 0.7237 - val_loss: 0.9582
Epoch 28/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 146s 210ms/step - accuracy: 0.9096 - loss: 0.2636 - val_accuracy: 0.7075 - val_loss: 1.0520
Epoch 29/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 146s 210ms/step - accuracy: 0.9284 - loss: 0.2130 - val_accuracy: 0.6986 - val_loss: 1.1467
Epoch 30/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 0.9382 - loss: 0.1868 - val_accuracy: 0.7030 - val_loss: 1.2158
Epoch 31/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 152s 219ms/step - accuracy: 0.9511 - loss: 0.1496 - val_accuracy: 0.6605 - val_loss: 1.5125
Epoch 32/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 152s 218ms/step - accuracy: 0.9567 - loss: 0.1363 - val_accuracy: 0.7078 - val_loss: 1.3059
Epoch 33/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 0.9666 - loss: 0.1038 - val_accuracy: 0.6959 - val_loss: 1.3778
Epoch 34/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 214ms/step - accuracy: 0.9750 - loss: 0.0813 - val_accuracy: 0.7129 - val_loss: 1.3267
Epoch 35/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 213ms/step - accuracy: 0.9832 - loss: 0.0631 - val_accuracy: 0.7012 - val_loss: 1.5128
Epoch 36/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 147s 211ms/step - accuracy: 0.9903 - loss: 0.0400 - val_accuracy: 0.7105 - val_loss: 1.4599
Epoch 37/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 147s 211ms/step - accuracy: 0.9886 - loss: 0.0431 - val_accuracy: 0.7111 - val_loss: 1.5320
Epoch 38/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 213ms/step - accuracy: 0.9915 - loss: 0.0332 - val_accuracy: 0.7041 - val_loss: 1.5704
Epoch 39/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 147s 211ms/step - accuracy: 0.9912 - loss: 0.0357 - val_accuracy: 0.7186 - val_loss: 1.6742
Epoch 40/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 147s 212ms/step - accuracy: 0.9984 - loss: 0.0106 - val_accuracy: 0.7141 - val_loss: 1.8869
Epoch 41/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 212ms/step - accuracy: 0.9992 - loss: 0.0068 - val_accuracy: 0.7294 - val_loss: 1.7659
Epoch 42/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 212ms/step - accuracy: 1.0000 - loss: 0.0024 - val_accuracy: 0.7294 - val_loss: 1.8203
Epoch 43/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 147s 212ms/step - accuracy: 1.0000 - loss: 0.0016 - val_accuracy: 0.7293 - val_loss: 1.8748
Epoch 44/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 147s 212ms/step - accuracy: 1.0000 - loss: 0.0013 - val_accuracy: 0.7302 - val_loss: 1.9234
Epoch 45/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 147s 212ms/step - accuracy: 1.0000 - loss: 0.0011 - val_accuracy: 0.7285 - val_loss: 1.9438
Epoch 46/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 147s 212ms/step - accuracy: 1.0000 - loss: 8.6068e-04 - val_accuracy: 0.7294 - val_loss: 1.9731
Epoch 47/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 212ms/step - accuracy: 1.0000 - loss: 7.9501e-04 - val_accuracy: 0.7302 - val_loss: 1.9970
Epoch 48/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 213ms/step - accuracy: 1.0000 - loss: 6.8612e-04 - val_accuracy: 0.7280 - val_loss: 2.0166
Epoch 49/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 212ms/step - accuracy: 1.0000 - loss: 6.3592e-04 - val_accuracy: 0.7305 - val_loss: 2.0381
Epoch 50/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 213ms/step - accuracy: 1.0000 - loss: 5.8269e-04 - val_accuracy: 0.7302 - val_loss: 2.0523
Epoch 51/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 214ms/step - accuracy: 1.0000 - loss: 5.4480e-04 - val_accuracy: 0.7293 - val_loss: 2.0756
Epoch 52/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 213ms/step - accuracy: 1.0000 - loss: 5.2392e-04 - val_accuracy: 0.7303 - val_loss: 2.0875
Epoch 53/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 213ms/step - accuracy: 1.0000 - loss: 4.7172e-04 - val_accuracy: 0.7300 - val_loss: 2.1021
Epoch 54/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 214ms/step - accuracy: 1.0000 - loss: 4.4116e-04 - val_accuracy: 0.7293 - val_loss: 2.1169
Epoch 55/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 4.1852e-04 - val_accuracy: 0.7285 - val_loss: 2.1296
Epoch 56/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 214ms/step - accuracy: 1.0000 - loss: 4.0114e-04 - val_accuracy: 0.7296 - val_loss: 2.1414
Epoch 57/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 213ms/step - accuracy: 1.0000 - loss: 3.8346e-04 - val_accuracy: 0.7294 - val_loss: 2.1552
Epoch 58/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 148s 214ms/step - accuracy: 1.0000 - loss: 3.5856e-04 - val_accuracy: 0.7293 - val_loss: 2.1648
Epoch 59/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 214ms/step - accuracy: 1.0000 - loss: 3.3809e-04 - val_accuracy: 0.7303 - val_loss: 2.1762
Epoch 60/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 3.2251e-04 - val_accuracy: 0.7298 - val_loss: 2.1840
Epoch 61/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 214ms/step - accuracy: 1.0000 - loss: 3.1467e-04 - val_accuracy: 0.7293 - val_loss: 2.1942
Epoch 62/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 214ms/step - accuracy: 1.0000 - loss: 3.0370e-04 - val_accuracy: 0.7302 - val_loss: 2.2038
Epoch 63/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 214ms/step - accuracy: 1.0000 - loss: 2.9600e-04 - val_accuracy: 0.7294 - val_loss: 2.2149
Epoch 64/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 214ms/step - accuracy: 1.0000 - loss: 2.7410e-04 - val_accuracy: 0.7300 - val_loss: 2.2233
Epoch 65/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 215ms/step - accuracy: 1.0000 - loss: 2.6988e-04 - val_accuracy: 0.7300 - val_loss: 2.2308
Epoch 66/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 215ms/step - accuracy: 1.0000 - loss: 2.5576e-04 - val_accuracy: 0.7293 - val_loss: 2.2417
Epoch 67/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 215ms/step - accuracy: 1.0000 - loss: 2.4013e-04 - val_accuracy: 0.7303 - val_loss: 2.2487
Epoch 68/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 215ms/step - accuracy: 1.0000 - loss: 2.4072e-04 - val_accuracy: 0.7287 - val_loss: 2.2552
Epoch 69/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 214ms/step - accuracy: 1.0000 - loss: 2.3353e-04 - val_accuracy: 0.7285 - val_loss: 2.2608
Epoch 70/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 215ms/step - accuracy: 1.0000 - loss: 2.2834e-04 - val_accuracy: 0.7307 - val_loss: 2.2740
Epoch 71/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 215ms/step - accuracy: 1.0000 - loss: 2.2156e-04 - val_accuracy: 0.7296 - val_loss: 2.2796
Epoch 72/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 214ms/step - accuracy: 1.0000 - loss: 2.1231e-04 - val_accuracy: 0.7284 - val_loss: 2.2853
Epoch 73/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 152s 218ms/step - accuracy: 1.0000 - loss: 2.0577e-04 - val_accuracy: 0.7291 - val_loss: 2.2926
Epoch 74/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 215ms/step - accuracy: 1.0000 - loss: 2.0056e-04 - val_accuracy: 0.7291 - val_loss: 2.2996
Epoch 75/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 215ms/step - accuracy: 1.0000 - loss: 1.9569e-04 - val_accuracy: 0.7291 - val_loss: 2.3054
Epoch 76/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 215ms/step - accuracy: 1.0000 - loss: 1.9288e-04 - val_accuracy: 0.7302 - val_loss: 2.3138
Epoch 77/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 215ms/step - accuracy: 1.0000 - loss: 1.8392e-04 - val_accuracy: 0.7293 - val_loss: 2.3181
Epoch 78/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.8316e-04 - val_accuracy: 0.7291 - val_loss: 2.3260
Epoch 79/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.7771e-04 - val_accuracy: 0.7300 - val_loss: 2.3299
Epoch 80/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 215ms/step - accuracy: 1.0000 - loss: 1.7198e-04 - val_accuracy: 0.7294 - val_loss: 2.3357
Epoch 81/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.6677e-04 - val_accuracy: 0.7291 - val_loss: 2.3412
Epoch 82/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 149s 215ms/step - accuracy: 1.0000 - loss: 1.6679e-04 - val_accuracy: 0.7291 - val_loss: 2.3493
Epoch 83/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 215ms/step - accuracy: 1.0000 - loss: 1.5874e-04 - val_accuracy: 0.7291 - val_loss: 2.3525
Epoch 84/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 1.5603e-04 - val_accuracy: 0.7291 - val_loss: 2.3584
Epoch 85/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 218ms/step - accuracy: 1.0000 - loss: 1.5137e-04 - val_accuracy: 0.7293 - val_loss: 2.3631
Epoch 86/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.5312e-04 - val_accuracy: 0.7293 - val_loss: 2.3703
Epoch 87/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 1.4811e-04 - val_accuracy: 0.7289 - val_loss: 2.3749
Epoch 88/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 215ms/step - accuracy: 1.0000 - loss: 1.4909e-04 - val_accuracy: 0.7291 - val_loss: 2.3793
Epoch 89/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 1.4342e-04 - val_accuracy: 0.7293 - val_loss: 2.3845
Epoch 90/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.3593e-04 - val_accuracy: 0.7291 - val_loss: 2.3880
Epoch 91/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.3309e-04 - val_accuracy: 0.7291 - val_loss: 2.3934
Epoch 92/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.3302e-04 - val_accuracy: 0.7291 - val_loss: 2.3986
Epoch 93/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.3011e-04 - val_accuracy: 0.7291 - val_loss: 2.4023
Epoch 94/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.2825e-04 - val_accuracy: 0.7289 - val_loss: 2.4094
Epoch 95/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.2747e-04 - val_accuracy: 0.7282 - val_loss: 2.4108
Epoch 96/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.2362e-04 - val_accuracy: 0.7291 - val_loss: 2.4158
Epoch 97/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 1.2297e-04 - val_accuracy: 0.7293 - val_loss: 2.4204
Epoch 98/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.1612e-04 - val_accuracy: 0.7287 - val_loss: 2.4246
Epoch 99/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 1.2015e-04 - val_accuracy: 0.7293 - val_loss: 2.4285
Epoch 100/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 215ms/step - accuracy: 1.0000 - loss: 1.1521e-04 - val_accuracy: 0.7282 - val_loss: 2.4319
Epoch 101/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 1.1093e-04 - val_accuracy: 0.7285 - val_loss: 2.4364
Epoch 102/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 152s 218ms/step - accuracy: 1.0000 - loss: 1.1191e-04 - val_accuracy: 0.7285 - val_loss: 2.4397
Epoch 103/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 153s 220ms/step - accuracy: 1.0000 - loss: 1.0994e-04 - val_accuracy: 0.7293 - val_loss: 2.4448
Epoch 104/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 153s 221ms/step - accuracy: 1.0000 - loss: 1.0857e-04 - val_accuracy: 0.7289 - val_loss: 2.4475
Epoch 105/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 154s 222ms/step - accuracy: 1.0000 - loss: 1.0643e-04 - val_accuracy: 0.7289 - val_loss: 2.4522
Epoch 106/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 153s 220ms/step - accuracy: 1.0000 - loss: 1.0293e-04 - val_accuracy: 0.7293 - val_loss: 2.4560
Epoch 107/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 156s 224ms/step - accuracy: 1.0000 - loss: 1.0100e-04 - val_accuracy: 0.7287 - val_loss: 2.4595
Epoch 108/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 155s 223ms/step - accuracy: 1.0000 - loss: 1.0090e-04 - val_accuracy: 0.7289 - val_loss: 2.4636
Epoch 109/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 156s 225ms/step - accuracy: 1.0000 - loss: 9.8881e-05 - val_accuracy: 0.7289 - val_loss: 2.4671
Epoch 110/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 154s 222ms/step - accuracy: 1.0000 - loss: 1.0021e-04 - val_accuracy: 0.7289 - val_loss: 2.4707
Epoch 111/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 154s 222ms/step - accuracy: 1.0000 - loss: 9.6634e-05 - val_accuracy: 0.7289 - val_loss: 2.4747
Epoch 112/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 154s 222ms/step - accuracy: 1.0000 - loss: 9.7639e-05 - val_accuracy: 0.7287 - val_loss: 2.4775
Epoch 113/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 159s 229ms/step - accuracy: 1.0000 - loss: 9.3657e-05 - val_accuracy: 0.7294 - val_loss: 2.4818
Epoch 114/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 155s 223ms/step - accuracy: 1.0000 - loss: 9.3266e-05 - val_accuracy: 0.7289 - val_loss: 2.4841
Epoch 115/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 9.0806e-05 - val_accuracy: 0.7287 - val_loss: 2.4870
Epoch 116/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 166s 239ms/step - accuracy: 1.0000 - loss: 9.0598e-05 - val_accuracy: 0.7287 - val_loss: 2.4900
Epoch 117/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 160s 230ms/step - accuracy: 1.0000 - loss: 8.9302e-05 - val_accuracy: 0.7294 - val_loss: 2.4942
Epoch 118/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 160s 230ms/step - accuracy: 1.0000 - loss: 8.8993e-05 - val_accuracy: 0.7282 - val_loss: 2.4971
Epoch 119/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 159s 229ms/step - accuracy: 1.0000 - loss: 8.4763e-05 - val_accuracy: 0.7285 - val_loss: 2.5007
Epoch 120/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 160s 230ms/step - accuracy: 1.0000 - loss: 8.6394e-05 - val_accuracy: 0.7285 - val_loss: 2.5049
Epoch 121/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 153s 220ms/step - accuracy: 1.0000 - loss: 8.6057e-05 - val_accuracy: 0.7287 - val_loss: 2.5073
Epoch 122/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 8.4339e-05 - val_accuracy: 0.7284 - val_loss: 2.5097
Epoch 123/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 8.3714e-05 - val_accuracy: 0.7289 - val_loss: 2.5144
Epoch 124/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 154s 221ms/step - accuracy: 1.0000 - loss: 8.1398e-05 - val_accuracy: 0.7285 - val_loss: 2.5159
Epoch 125/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 8.0337e-05 - val_accuracy: 0.7287 - val_loss: 2.5202
Epoch 126/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 7.7960e-05 - val_accuracy: 0.7282 - val_loss: 2.5227
Epoch 127/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 7.8509e-05 - val_accuracy: 0.7282 - val_loss: 2.5245
Epoch 128/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 7.8316e-05 - val_accuracy: 0.7284 - val_loss: 2.5282
Epoch 129/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 7.6893e-05 - val_accuracy: 0.7291 - val_loss: 2.5316
Epoch 130/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 152s 219ms/step - accuracy: 1.0000 - loss: 7.6874e-05 - val_accuracy: 0.7285 - val_loss: 2.5341
Epoch 131/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 152s 218ms/step - accuracy: 1.0000 - loss: 7.3727e-05 - val_accuracy: 0.7291 - val_loss: 2.5366
Epoch 132/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 153s 220ms/step - accuracy: 1.0000 - loss: 7.3063e-05 - val_accuracy: 0.7282 - val_loss: 2.5392
Epoch 133/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 7.4919e-05 - val_accuracy: 0.7284 - val_loss: 2.5432
Epoch 134/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 7.0851e-05 - val_accuracy: 0.7284 - val_loss: 2.5446
Epoch 135/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 7.1533e-05 - val_accuracy: 0.7287 - val_loss: 2.5471
Epoch 136/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 7.0703e-05 - val_accuracy: 0.7285 - val_loss: 2.5505
Epoch 137/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 7.0234e-05 - val_accuracy: 0.7291 - val_loss: 2.5530
Epoch 138/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 218ms/step - accuracy: 1.0000 - loss: 6.9904e-05 - val_accuracy: 0.7285 - val_loss: 2.5565
Epoch 139/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 6.6885e-05 - val_accuracy: 0.7289 - val_loss: 2.5583
Epoch 140/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 6.7599e-05 - val_accuracy: 0.7285 - val_loss: 2.5613
Epoch 141/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 6.7909e-05 - val_accuracy: 0.7287 - val_loss: 2.5635
Epoch 142/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 152s 219ms/step - accuracy: 1.0000 - loss: 6.7638e-05 - val_accuracy: 0.7291 - val_loss: 2.5656
Epoch 143/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 6.5869e-05 - val_accuracy: 0.7293 - val_loss: 2.5678
Epoch 144/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 6.5031e-05 - val_accuracy: 0.7282 - val_loss: 2.5706
Epoch 145/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 6.4927e-05 - val_accuracy: 0.7285 - val_loss: 2.5736
Epoch 146/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 6.3302e-05 - val_accuracy: 0.7285 - val_loss: 2.5760
Epoch 147/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 6.5542e-05 - val_accuracy: 0.7287 - val_loss: 2.5776
Epoch 148/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 150s 216ms/step - accuracy: 1.0000 - loss: 6.1796e-05 - val_accuracy: 0.7287 - val_loss: 2.5805
Epoch 149/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 151s 217ms/step - accuracy: 1.0000 - loss: 6.1350e-05 - val_accuracy: 0.7289 - val_loss: 2.5828
Epoch 150/150
695/695 ━━━━━━━━━━━━━━━━━━━━ 152s 219ms/step - accuracy: 1.0000 - loss: 6.1446e-05 - val_accuracy: 0.7282 - val_loss: 2.5856
313/313 - 11s - 35ms/step - accuracy: 0.7250 - loss: 2.6708
Test accuracy: 0.7250000238418579

plt.figure(figsize=(10, 6))
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.axhline(y=test_acc, c='r',  linestyle='--', label=f'Test Accuracy of ANN: {test_acc:.4f}')
plt.axhline(y=test_acc1, c='g' ,linestyle='--', label=f'Test Accuracy of CNN: {test_acc1:.4f}')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.title('Training and Validation Accuracy over Epochs')
plt.legend()
plt.grid(True)
plt.show()

Experiment 2.5: Understand the effect of weight initialization technique on models’ performance.

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models, initializers
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

def create_ann_model(initializer):
    model = models.Sequential([
        layers.Flatten(input_shape=(32, 32, 3)),
        layers.Dense(128, activation='relu', kernel_initializer=initializer),
        layers.Dense(64, activation='relu', kernel_initializer=initializer),
        layers.Dense(10, activation='softmax', kernel_initializer=initializer)
    ])
    model.compile(optimizer='sgd',
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])
    return model

initializers_list = {
    'Zero Initialization': initializers.Zeros(),
    'Random Normal Initialization': initializers.RandomNormal(mean=0.0, stddev=1.0, seed=42),
    'Random Uniform Initialization': initializers.RandomUniform(minval=0.0, maxval=1.0, seed=42),
    'Xavier/Glorot Uniform Initialization': initializers.GlorotUniform(seed=42),
    'He Uniform Initialization': initializers.HeUniform(seed=42),
    'Xavier/Glorot Normal Initialization': initializers.GlorotNormal(seed=42),
    'He Normal Initialization': initializers.HeNormal(seed=42)
}

initializer_names = []
val_accuracies = []
test_accuracies = []

for name, initializer in initializers_list.items():
    model = create_ann_model(initializer)
    history = model.fit(train_images, train_labels, epochs=15, validation_data=(val_images, val_labels), verbose=0)
    val_acc = max(history.history['val_accuracy'])
    test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)
    initializer_names.append(name)
    val_accuracies.append(val_acc)
    test_accuracies.append(test_acc)
    print(f'{name}, Validation Accuracy: {val_acc}, Test Accuracy: {test_acc}')

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Zero Initialization, Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612
Random Normal Initialization, Validation Accuracy: 0.1004500463604927, Test Accuracy: 0.10010000318288803
Random Uniform Initialization, Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612
Xavier/Glorot Uniform Initialization, Validation Accuracy: 0.49504950642585754, Test Accuracy: 0.4912000000476837
He Uniform Initialization, Validation Accuracy: 0.5013501644134521, Test Accuracy: 0.5005000233650208
Xavier/Glorot Normal Initialization, Validation Accuracy: 0.4982898235321045, Test Accuracy: 0.5023000240325928
He Normal Initialization, Validation Accuracy: 0.48352834582328796, Test Accuracy: 0.46939998865127563

fig, ax = plt.subplots(2, 1, figsize=(12, 12))

ax[0].bar(initializer_names, val_accuracies, color='green')
ax[0].set_title('Validation Accuracy vs. Weight Initialization Techniques over 15 Epochs')
ax[0].set_xlabel('Weight Initialization Techniques')
ax[0].set_ylabel('Validation Accuracy')
ax[0].set_xticks(range(len(initializer_names)))
ax[0].set_xticklabels(initializer_names, rotation=45)
ax[0].grid(True)

ax[1].bar(initializer_names, test_accuracies, color='lightred')
ax[1].set_title('Test Accuracy vs. Weight Initialization Techniques over 15 Epochs')
ax[1].set_xlabel('Weight Initialization Techniques')
ax[1].set_ylabel('Test Accuracy')
ax[1].set_xticks(range(len(initializer_names)))
ax[1].set_xticklabels(initializer_names, rotation=45)
ax[1].grid(True)

plt.tight_layout()
plt.show()

Experiment 2.6: Understand the role of Optimization Algorithm and Implement the different Algorithms starting with Batch Gradient Descent, SGD, mini-batch SGD, Momentum, RMS prop and ADAM. Also study the role of Learning rate.

Experiment to determine the effect on models performance when using different optimization algorithms at learning rate 0.01

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models, optimizers
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

def create_cnn_model():
    model = models.Sequential([
        layers.Flatten(input_shape=(32, 32, 3)),
        layers.Dense(256, activation='relu'),
        layers.Dense(128, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(10, activation='softmax')
    ])
    return model

def compile_and_train_model(optimizer, epochs=15, batch_size=64):
    model = create_cnn_model()
    model.compile(optimizer=optimizer,
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    history = model.fit(train_images, train_labels, epochs=epochs, batch_size=batch_size,
                        validation_data=(val_images, val_labels), verbose=0)

    val_acc = max(history.history['val_accuracy'])
    test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)

    return val_acc, test_acc

optimizers_list = {
    'Batch Gradient Descent': (optimizers.SGD(learning_rate=0.01), len(train_images)),  
    'SGD': (optimizers.SGD(learning_rate=0.01), 1),  
    'Mini-batch SGD': (optimizers.SGD(learning_rate=0.01), 64),  
    'Momentum': (optimizers.SGD(learning_rate=0.01, momentum=0.9), 64),
    'RMSprop': (optimizers.RMSprop(learning_rate=0.01), 64),
    'Adam': (optimizers.Adam(learning_rate=0.01), 64)
}

optimizer_names = []
val_accuracies = []
test_accuracies = []

for name, (optimizer, batch_size) in optimizers_list.items():
    val_acc, test_acc = compile_and_train_model(optimizer, batch_size=batch_size)
    optimizer_names.append(name)
    val_accuracies.append(val_acc)
    test_accuracies.append(test_acc)
    print(f'{name}, Validation Accuracy: {val_acc}, Test Accuracy: {test_acc}')

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Batch Gradient Descent, Validation Accuracy: 0.13447344303131104, Test Accuracy: 0.13609999418258667
SGD, Validation Accuracy: 0.4063006341457367, Test Accuracy: 0.4041999876499176
Mini-batch SGD, Validation Accuracy: 0.4660665988922119, Test Accuracy: 0.39980000257492065
Momentum, Validation Accuracy: 0.49774977564811707, Test Accuracy: 0.49230000376701355
RMSprop, Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612
Adam, Validation Accuracy: 0.3670566976070404, Test Accuracy: 0.3361999988555908

fig, ax = plt.subplots(2, 1, figsize=(12, 12))

ax[0].bar(optimizer_names, val_accuracies, color='red')
ax[0].set_title('Validation Accuracy vs. Optimization Algorithms over 15 Epochs at learning rate 0.01')
ax[0].set_xlabel('Optimization Algorithms')
ax[0].set_ylabel('Validation Accuracy')
ax[0].set_xticks(range(len(optimizer_names)))
ax[0].set_xticklabels(optimizer_names, rotation=45)
ax[0].grid(True)

ax[1].bar(optimizer_names, test_accuracies, color='pink')
ax[1].set_title('Test Accuracy vs. Optimization Algorithms over 15 Epochs at learning rate 0.01')
ax[1].set_xlabel('Optimization Algorithms')
ax[1].set_ylabel('Test Accuracy')
ax[1].set_xticks(range(len(optimizer_names)))
ax[1].set_xticklabels(optimizer_names, rotation=45)
ax[1].grid(True)

plt.tight_layout()
plt.show()

Experiment to determine the effect on models performance when using different optimization algorithms at learning rate 0.001

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models, optimizers
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

def create_cnn_model():
    model = models.Sequential([
        layers.Flatten(input_shape=(32, 32, 3)),
        layers.Dense(256, activation='relu'),
        layers.Dense(128, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(10, activation='softmax')
    ])
    return model

def compile_and_train_model(optimizer, epochs=20, batch_size=64):
    model = create_cnn_model()
    model.compile(optimizer=optimizer,
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    history = model.fit(train_images, train_labels, epochs=epochs, batch_size=batch_size,
                        validation_data=(val_images, val_labels), verbose=0)

    val_acc = max(history.history['val_accuracy'])
    test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)

    return val_acc, test_acc

optimizers_list = {
    'Batch Gradient Descent': (optimizers.SGD(learning_rate=0.001), len(train_images)),
    'SGD': (optimizers.SGD(learning_rate=0.001), 1),
    'Mini-batch SGD': (optimizers.SGD(learning_rate=0.001), 64),
    'Momentum': (optimizers.SGD(learning_rate=0.001, momentum=0.9), 64),
    'RMSprop': (optimizers.RMSprop(learning_rate=0.001), 64),
    'Adam': (optimizers.Adam(learning_rate=0.001), 64)
}

optimizer_names = []
val_accuracies = []
test_accuracies = []

for name, (optimizer, batch_size) in optimizers_list.items():
    val_acc, test_acc = compile_and_train_model(optimizer, batch_size=batch_size)
    optimizer_names.append(name)
    val_accuracies.append(val_acc)
    test_accuracies.append(test_acc)
    print(f'{name}, Validation Accuracy: {val_acc}, Test Accuracy: {test_acc}')

fig, ax = plt.subplots(2, 1, figsize=(12, 12))

ax[0].bar(optimizer_names, val_accuracies, color='skyblue')
ax[0].set_title('Validation Accuracy vs. Optimization Algorithms')
ax[0].set_xlabel('Optimization Algorithms')
ax[0].set_ylabel('Validation Accuracy')
ax[0].set_xticks(range(len(optimizer_names)))
ax[0].set_xticklabels(optimizer_names, rotation=45)
ax[0].grid(True)

ax[1].bar(optimizer_names, test_accuracies, color='lightgreen')
ax[1].set_title('Test Accuracy vs. Optimization Algorithms')
ax[1].set_xlabel('Optimization Algorithms')
ax[1].set_ylabel('Test Accuracy')
ax[1].set_xticks(range(len(optimizer_names)))
ax[1].set_xticklabels(optimizer_names, rotation=45)
ax[1].grid(True)

plt.tight_layout()
plt.show()

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Batch Gradient Descent, Validation Accuracy: 0.09666966646909714, Test Accuracy: 0.09070000052452087
SGD, Validation Accuracy: 0.521692156791687, Test Accuracy: 0.5120999813079834
Mini-batch SGD, Validation Accuracy: 0.4185418486595154, Test Accuracy: 0.4226999878883362
Momentum, Validation Accuracy: 0.502970278263092, Test Accuracy: 0.4999000132083893
RMSprop, Validation Accuracy: 0.4775877594947815, Test Accuracy: 0.4740999937057495
Adam, Validation Accuracy: 0.49558955430984497, Test Accuracy: 0.49720001220703125

Experiment to determine the effect on models performance when using different optimization algorithms at learning rate 0.1

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models, optimizers
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

def create_cnn_model():
    model = models.Sequential([
        layers.Flatten(input_shape=(32, 32, 3)),
        layers.Dense(256, activation='relu'),
        layers.Dense(128, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(10, activation='softmax')
    ])
    return model

def compile_and_train_model(optimizer, epochs=10, batch_size=64):
    model = create_cnn_model()
    model.compile(optimizer=optimizer,
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    history = model.fit(train_images, train_labels, epochs=epochs, batch_size=batch_size,
                        validation_data=(val_images, val_labels), verbose=0)

    val_acc = max(history.history['val_accuracy'])
    test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)

    return val_acc, test_acc

l = 0.1  

optimizers_list = {
    'Batch Gradient Descent': (optimizers.SGD(learning_rate=l), len(train_images)),  
    'SGD': (optimizers.SGD(learning_rate=l), 1),  
    'Mini-batch SGD': (optimizers.SGD(learning_rate=l), 64),  
    'Momentum': (optimizers.SGD(learning_rate=l, momentum=0.9), 64),
    'RMSprop': (optimizers.RMSprop(learning_rate=l), 64),
    'Adam': (optimizers.Adam(learning_rate=l), 64)
}

optimizer_names = []
val_accuracies = []
test_accuracies = []

for name, (optimizer, batch_size) in optimizers_list.items():
    val_acc, test_acc = compile_and_train_model(optimizer, batch_size=batch_size)
    optimizer_names.append(name)
    val_accuracies.append(val_acc)
    test_accuracies.append(test_acc)
    print(f'{name}, Validation Accuracy: {val_acc}, Test Accuracy: {test_acc}')

fig, ax = plt.subplots(2, 1, figsize=(12, 12))

ax[0].bar(optimizer_names, val_accuracies, color='skyblue')
ax[0].set_title('Validation Accuracy vs. Optimization Algorithms')
ax[0].set_xlabel('Optimization Algorithms')
ax[0].set_ylabel('Validation Accuracy')
ax[0].set_xticks(range(len(optimizer_names)))
ax[0].set_xticklabels(optimizer_names, rotation=45)
ax[0].grid(True)

ax[1].bar(optimizer_names, test_accuracies, color='lightgreen')
ax[1].set_title('Test Accuracy vs. Optimization Algorithms')
ax[1].set_xlabel('Optimization Algorithms')
ax[1].set_ylabel('Test Accuracy')
ax[1].set_xticks(range(len(optimizer_names)))
ax[1].set_xticklabels(optimizer_names, rotation=45)
ax[1].grid(True)

plt.tight_layout()
plt.show()

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Batch Gradient Descent, Validation Accuracy: 0.17137713730335236, Test Accuracy: 0.17090000212192535
SGD, Validation Accuracy: 0.09990999102592468, Test Accuracy: 0.10000000149011612
Mini-batch SGD, Validation Accuracy: 0.4498649835586548, Test Accuracy: 0.4339999854564667
Momentum, Validation Accuracy: 0.16651664674282074, Test Accuracy: 0.10000000149011612
RMSprop, Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612
Adam, Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612

Experiment to determine the effect on models performance when using different optimization algorithms at learning rate 0.05

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models, optimizers
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

def create_cnn_model():
    model = models.Sequential([
        layers.Flatten(input_shape=(32, 32, 3)),
        layers.Dense(256, activation='relu'),
        layers.Dense(128, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(10, activation='softmax')
    ])
    return model

def compile_and_train_model(optimizer, epochs=10, batch_size=64):
    model = create_cnn_model()
    model.compile(optimizer=optimizer,
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    history = model.fit(train_images, train_labels, epochs=epochs, batch_size=batch_size,
                        validation_data=(val_images, val_labels), verbose=0)

    val_acc = max(history.history['val_accuracy'])
    test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)

    return val_acc, test_acc

l = 0.05

optimizers_list = {
    'Batch Gradient Descent': (optimizers.SGD(learning_rate=l), len(train_images)),
    'SGD': (optimizers.SGD(learning_rate=l), 1),
    'Mini-batch SGD': (optimizers.SGD(learning_rate=l), 64),
    'Momentum': (optimizers.SGD(learning_rate=l, momentum=0.9), 64),
    'RMSprop': (optimizers.RMSprop(learning_rate=l), 64),
    'Adam': (optimizers.Adam(learning_rate=l), 64)
}

optimizer_names = []
val_accuracies = []
test_accuracies = []

for name, (optimizer, batch_size) in optimizers_list.items():
    val_acc, test_acc = compile_and_train_model(optimizer, batch_size=batch_size)
    optimizer_names.append(name)
    val_accuracies.append(val_acc)
    test_accuracies.append(test_acc)
    print(f'{name}, Validation Accuracy: {val_acc}, Test Accuracy: {test_acc}')

fig, ax = plt.subplots(2, 1, figsize=(12, 12))

ax[0].bar(optimizer_names, val_accuracies, color='skyblue')
ax[0].set_title('Validation Accuracy vs. Optimization Algorithms')
ax[0].set_xlabel('Optimization Algorithms')
ax[0].set_ylabel('Validation Accuracy')
ax[0].set_xticks(range(len(optimizer_names)))
ax[0].set_xticklabels(optimizer_names, rotation=45)
ax[0].grid(True)

ax[1].bar(optimizer_names, test_accuracies, color='lightgreen')
ax[1].set_title('Test Accuracy vs. Optimization Algorithms')
ax[1].set_xlabel('Optimization Algorithms')
ax[1].set_ylabel('Test Accuracy')
ax[1].set_xticks(range(len(optimizer_names)))
ax[1].set_xticklabels(optimizer_names, rotation=45)
ax[1].grid(True)

plt.tight_layout()
plt.show()

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Batch Gradient Descent, Validation Accuracy: 0.17875787615776062, Test Accuracy: 0.1776999980211258
SGD, Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612
Mini-batch SGD, Validation Accuracy: 0.48280829191207886, Test Accuracy: 0.47620001435279846
Momentum, Validation Accuracy: 0.38793879747390747, Test Accuracy: 0.3804999887943268
RMSprop, Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612
Adam, Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612

Experiment to determine the effect on models performance when using different optimization algorithms at learning rate 0.009

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models, optimizers
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

def create_cnn_model():
    model = models.Sequential([
        layers.Flatten(input_shape=(32, 32, 3)),
        layers.Dense(256, activation='relu'),
        layers.Dense(128, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(10, activation='softmax')
    ])
    return model

def compile_and_train_model(optimizer, epochs=10, batch_size=64):
    model = create_cnn_model()
    model.compile(optimizer=optimizer,
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    history = model.fit(train_images, train_labels, epochs=epochs, batch_size=batch_size,
                        validation_data=(val_images, val_labels), verbose=0)

    val_acc = max(history.history['val_accuracy'])
    test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)

    return val_acc, test_acc

l = 0.009

optimizers_list = {
    'Batch Gradient Descent': (optimizers.SGD(learning_rate=l), len(train_images)),
    'SGD': (optimizers.SGD(learning_rate=l), 1),
    'Mini-batch SGD': (optimizers.SGD(learning_rate=l), 64),
    'Momentum': (optimizers.SGD(learning_rate=l, momentum=0.9), 64),
    'RMSprop': (optimizers.RMSprop(learning_rate=l), 64),
    'Adam': (optimizers.Adam(learning_rate=l), 64)
}

optimizer_names = []
val_accuracies = []
test_accuracies = []

for name, (optimizer, batch_size) in optimizers_list.items():
    val_acc, test_acc = compile_and_train_model(optimizer, batch_size=batch_size)
    optimizer_names.append(name)
    val_accuracies.append(val_acc)
    test_accuracies.append(test_acc)
    print(f'{name}, Validation Accuracy: {val_acc}, Test Accuracy: {test_acc}')

fig, ax = plt.subplots(2, 1, figsize=(12, 12))

ax[0].bar(optimizer_names, val_accuracies, color='skyblue')
ax[0].set_title('Validation Accuracy vs. Optimization Algorithms')
ax[0].set_xlabel('Optimization Algorithms')
ax[0].set_ylabel('Validation Accuracy')
ax[0].set_xticks(range(len(optimizer_names)))
ax[0].set_xticklabels(optimizer_names, rotation=45)
ax[0].grid(True)

ax[1].bar(optimizer_names, test_accuracies, color='lightgreen')
ax[1].set_title('Test Accuracy vs. Optimization Algorithms')
ax[1].set_xlabel('Optimization Algorithms')
ax[1].set_ylabel('Test Accuracy')
ax[1].set_xticks(range(len(optimizer_names)))
ax[1].set_xticklabels(optimizer_names, rotation=45)
ax[1].grid(True)

plt.tight_layout()
plt.show()

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Batch Gradient Descent, Validation Accuracy: 0.13447344303131104, Test Accuracy: 0.12929999828338623
SGD, Validation Accuracy: 0.41080108284950256, Test Accuracy: 0.4004000127315521
Mini-batch SGD, Validation Accuracy: 0.44950494170188904, Test Accuracy: 0.4526999890804291
Momentum, Validation Accuracy: 0.47146713733673096, Test Accuracy: 0.4758000075817108
RMSprop, Validation Accuracy: 0.2145814597606659, Test Accuracy: 0.20360000431537628
Adam, Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612

Experiment to determine models training time when using different optimization algorithms at learning rate 0.009 for just one Epoch.

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models, optimizers
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import time

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

def create_cnn_model():
    model = models.Sequential([
        layers.Flatten(input_shape=(32, 32, 3)),
        layers.Dense(256, activation='relu'),
        layers.Dense(128, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(10, activation='softmax')
    ])
    return model

def compile_and_train_model(optimizer, epochs=1, batch_size=64):
    model = create_cnn_model()
    model.compile(optimizer=optimizer,
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    start_time = time.time()
    history = model.fit(train_images, train_labels, epochs=epochs, batch_size=batch_size,
                        validation_data=(val_images, val_labels), verbose=0)
    end_time = time.time()
    
    val_acc = max(history.history['val_accuracy'])
    test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)
    training_time = end_time - start_time

    return val_acc, test_acc, training_time

learning_rate = 0.009
optimizers_list = {
    'Batch Gradient Descent': (optimizers.SGD(learning_rate=learning_rate), len(train_images)),
    'SGD': (optimizers.SGD(learning_rate=learning_rate), 1),
    'Mini-batch SGD': (optimizers.SGD(learning_rate=learning_rate), 64),
    'Momentum': (optimizers.SGD(learning_rate=learning_rate, momentum=0.9), 64),
    'RMSprop': (optimizers.RMSprop(learning_rate=learning_rate), 64),
    'Adam': (optimizers.Adam(learning_rate=learning_rate), 64)
}

optimizer_names = []
training_times = []

for name, (optimizer, batch_size) in optimizers_list.items():
    val_acc, test_acc, training_time = compile_and_train_model(optimizer, batch_size=batch_size)
    optimizer_names.append(name)
    training_times.append(training_time)
    print(f'{name}, Training Time: {training_time:.2f} seconds')

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Batch Gradient Descent, Training Time: 4.20 seconds
SGD, Training Time: 91.31 seconds
Mini-batch SGD, Training Time: 3.83 seconds
Momentum, Training Time: 4.76 seconds
RMSprop, Training Time: 5.48 seconds
Adam, Training Time: 6.73 seconds

fig, ax = plt.subplots(figsize=(12, 6))
ax.bar(optimizer_names, training_times, color='yellow')
ax.set_title('Training Time vs. Optimization Algorithms for One Epoch')
ax.set_xlabel('Optimization Algorithms')
ax.set_ylabel('Training Time (seconds)')
ax.set_xticks(range(len(optimizer_names)))
ax.set_xticklabels(optimizer_names, rotation=45)
ax.grid(True)

plt.tight_layout()
plt.show()

Experiment to determine models training time when using different optimization algorithms at learning rate 0.009 for 10 Epoch.

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models, optimizers
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import time

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

def create_cnn_model():
    model = models.Sequential([
        layers.Flatten(input_shape=(32, 32, 3)),
        layers.Dense(256, activation='relu'),
        layers.Dense(128, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(10, activation='softmax')
    ])
    return model

def compile_and_train_model(optimizer, epochs=10, batch_size=64):
    model = create_cnn_model()
    model.compile(optimizer=optimizer,
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    start_time = time.time()
    history = model.fit(train_images, train_labels, epochs=epochs, batch_size=batch_size,
                        validation_data=(val_images, val_labels), verbose=0)
    end_time = time.time()
    
    val_acc = max(history.history['val_accuracy'])
    test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)
    training_time = end_time - start_time

    return val_acc, test_acc, training_time

learning_rate = 0.009
optimizers_list = {
    'Batch Gradient Descent': (optimizers.SGD(learning_rate=learning_rate), len(train_images)),
    'SGD': (optimizers.SGD(learning_rate=learning_rate), 1),
    'Mini-batch SGD': (optimizers.SGD(learning_rate=learning_rate), 64),
    'Momentum': (optimizers.SGD(learning_rate=learning_rate, momentum=0.9), 64),
    'RMSprop': (optimizers.RMSprop(learning_rate=learning_rate), 64),
    'Adam': (optimizers.Adam(learning_rate=learning_rate), 64)
}

optimizer_names = []
training_times = []

for name, (optimizer, batch_size) in optimizers_list.items():
    val_acc, test_acc, training_time = compile_and_train_model(optimizer, batch_size=batch_size)
    optimizer_names.append(name)
    training_times.append(training_time)
    print(f'{name}, Training Time: {training_time:.2f} seconds')

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

<keras.src.optimizers.sgd.SGD object at 0x000002935DC3CB20> 44445
Batch Gradient Descent, Training Time: 9.00 seconds
<keras.src.optimizers.sgd.SGD object at 0x000002936A22DB40> 1
SGD, Training Time: 882.84 seconds
<keras.src.optimizers.sgd.SGD object at 0x000002936A22DDE0> 64
Mini-batch SGD, Training Time: 24.40 seconds
<keras.src.optimizers.sgd.SGD object at 0x000002936CD6AE00> 64
Momentum, Training Time: 27.49 seconds
<keras.src.optimizers.rmsprop.RMSprop object at 0x000002936D0D2FE0> 64
RMSprop, Training Time: 35.89 seconds
<keras.src.optimizers.adam.Adam object at 0x000002936D16FFA0> 64
Adam, Training Time: 43.13 seconds

# Plot the results
fig, ax = plt.subplots(figsize=(12, 6))

# Plotting the bar graph on the single axis
ax.bar(optimizer_names, training_times, color='lightcoral')
ax.set_title('Training Time vs. Optimization Algorithms for 10 Epoch')
ax.set_xlabel('Optimization Algorithms')
ax.set_ylabel('Training Time (seconds)')
ax.set_xticks(range(len(optimizer_names)))
ax.set_xticklabels(optimizer_names, rotation=45)
ax.grid(True)

plt.tight_layout()
plt.show()

Experiment to determine models performance, training time when using different optimization algorithms like SGD, Momentum, RMSprop, Adam at learning rates [0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6] for 25 Epoch.

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models, optimizers
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import time

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

train_images, val_images, train_labels, val_labels = train_test_split(
    train_images, train_labels, test_size=0.1111, random_state=42, stratify=train_labels)

def create_cnn_model():
    model = models.Sequential([
        layers.Flatten(input_shape=(32, 32, 3)),
        layers.Dense(256, activation='relu'),
        layers.Dense(128, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(10, activation='softmax')
    ])
    return model

def compile_and_train_model(optimizer, epochs=1, batch_size=64):
    model = create_cnn_model()
    model.compile(optimizer=optimizer,
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])
    
    start_time = time.time()
    history = model.fit(train_images, train_labels, epochs=epochs, batch_size=batch_size,
                        validation_data=(val_images, val_labels), verbose=0)
    training_time = time.time() - start_time

    val_acc = max(history.history['val_accuracy'])
    test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)

    return val_acc, test_acc, training_time, history

learning_rates = [0.001, 0.003, 0.006, 0.01, 0.03, 0.06,  0.1, 0.3, 0.6]
optimizers_list = {
    'SGD': [(optimizers.SGD(learning_rate=lr), lr) for lr in learning_rates],
    'Momentum': [(optimizers.SGD(learning_rate=lr, momentum=0.9), lr) for lr in learning_rates],
    'RMSprop': [(optimizers.RMSprop(learning_rate=lr), lr) for lr in learning_rates],
    'Adam': [(optimizers.Adam(learning_rate=lr), lr) for lr in learning_rates]
}

optimizer_names = []
val_accuracies = []
test_accuracies = []
training_times = []
histories = []

for name, optimizer_lr_list in optimizers_list.items():
    for optimizer, lr in optimizer_lr_list:
        val_acc, test_acc, training_time, history = compile_and_train_model(optimizer, epochs=25, batch_size=64)
        optimizer_names.append(f'{name} (lr={lr})')
        val_accuracies.append(val_acc)
        test_accuracies.append(test_acc)
        training_times.append(training_time)
        histories.append(history)
        print(f'{name} (lr={lr}), Validation Accuracy: {val_acc}, Test Accuracy: {test_acc}, Training Time: {training_time}')

fig, ax = plt.subplots(3, 1, figsize=(12, 18))

ax[0].bar(optimizer_names, val_accuracies, color='skyblue')
ax[0].set_title('Validation Accuracy vs. Optimization Algorithms and Learning Rates')
ax[0].set_xlabel('Optimization Algorithms and Learning Rates')
ax[0].set_ylabel('Validation Accuracy')
ax[0].set_xticks(range(len(optimizer_names)))
ax[0].set_xticklabels(optimizer_names, rotation=45)
ax[0].grid(True)

ax[1].bar(optimizer_names, test_accuracies, color='lightgreen')
ax[1].set_title('Test Accuracy vs. Optimization Algorithms and Learning Rates')
ax[1].set_xlabel('Optimization Algorithms and Learning Rates')
ax[1].set_ylabel('Test Accuracy')
ax[1].set_xticks(range(len(optimizer_names)))
ax[1].set_xticklabels(optimizer_names, rotation=45)
ax[1].grid(True)

ax[2].bar(optimizer_names, training_times, color='lightcoral')
ax[2].set_title('Training Time vs. Optimization Algorithms and Learning Rates')
ax[2].set_xlabel('Optimization Algorithms and Learning Rates')
ax[2].set_ylabel('Training Time (seconds)')
ax[2].set_xticks(range(len(optimizer_names)))
ax[2].set_xticklabels(optimizer_names, rotation=45)
ax[2].grid(True)

plt.tight_layout()
plt.show()

fig, ax = plt.subplots(figsize=(12, 8))

for name, history in zip(optimizer_names, histories):
    ax.plot(history.history['val_accuracy'], label=name)

ax.set_title('Validation Accuracy over Epochs')
ax.set_xlabel('Epoch')
ax.set_ylabel('Validation Accuracy')
ax.legend()
ax.grid(True)

plt.show()

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

SGD (lr=0.001), Validation Accuracy: 0.41170117259025574, Test Accuracy: 0.414900004863739, Training Time: 59.90916466712952
SGD (lr=0.003), Validation Accuracy: 0.46570655703544617, Test Accuracy: 0.4523000121116638, Training Time: 60.17952847480774
SGD (lr=0.006), Validation Accuracy: 0.507830798625946, Test Accuracy: 0.5127999782562256, Training Time: 60.077714681625366
SGD (lr=0.01), Validation Accuracy: 0.4959495961666107, Test Accuracy: 0.4984999895095825, Training Time: 59.67248606681824
SGD (lr=0.03), Validation Accuracy: 0.507830798625946, Test Accuracy: 0.41019999980926514, Training Time: 59.59853005409241
SGD (lr=0.06), Validation Accuracy: 0.4934293329715729, Test Accuracy: 0.5080000162124634, Training Time: 60.43995141983032
SGD (lr=0.1), Validation Accuracy: 0.49432942271232605, Test Accuracy: 0.503600001335144, Training Time: 60.526172399520874
SGD (lr=0.3), Validation Accuracy: 0.4498649835586548, Test Accuracy: 0.435699999332428, Training Time: 60.75846862792969
SGD (lr=0.6), Validation Accuracy: 0.09990999102592468, Test Accuracy: 0.10000000149011612, Training Time: 60.53550148010254
Momentum (lr=0.001), Validation Accuracy: 0.5067507028579712, Test Accuracy: 0.49900001287460327, Training Time: 67.16620874404907
Momentum (lr=0.003), Validation Accuracy: 0.5202520489692688, Test Accuracy: 0.5138000249862671, Training Time: 66.87788820266724
Momentum (lr=0.006), Validation Accuracy: 0.5117911696434021, Test Accuracy: 0.5076000094413757, Training Time: 66.39665341377258
Momentum (lr=0.01), Validation Accuracy: 0.5031503438949585, Test Accuracy: 0.5073000192642212, Training Time: 66.67020010948181
Momentum (lr=0.03), Validation Accuracy: 0.4617461860179901, Test Accuracy: 0.46140000224113464, Training Time: 66.64304995536804
Momentum (lr=0.06), Validation Accuracy: 0.3463546335697174, Test Accuracy: 0.29420000314712524, Training Time: 66.45746040344238
Momentum (lr=0.1), Validation Accuracy: 0.19441944360733032, Test Accuracy: 0.10000000149011612, Training Time: 66.6251471042633
Momentum (lr=0.3), Validation Accuracy: 0.09990999102592468, Test Accuracy: 0.10000000149011612, Training Time: 69.95335173606873
Momentum (lr=0.6), Validation Accuracy: 0.09990999102592468, Test Accuracy: 0.10000000149011612, Training Time: 66.88169050216675
RMSprop (lr=0.001), Validation Accuracy: 0.4901890158653259, Test Accuracy: 0.4869000017642975, Training Time: 92.40610194206238
RMSprop (lr=0.003), Validation Accuracy: 0.4212421178817749, Test Accuracy: 0.39969998598098755, Training Time: 89.84469389915466
RMSprop (lr=0.006), Validation Accuracy: 0.3463546335697174, Test Accuracy: 0.28209999203681946, Training Time: 90.36036109924316
RMSprop (lr=0.01), Validation Accuracy: 0.2653465270996094, Test Accuracy: 0.259799987077713, Training Time: 88.80380821228027
RMSprop (lr=0.03), Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612, Training Time: 88.7582426071167
RMSprop (lr=0.06), Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612, Training Time: 88.59473609924316
RMSprop (lr=0.1), Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612, Training Time: 88.38554430007935
RMSprop (lr=0.3), Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612, Training Time: 88.68279957771301
RMSprop (lr=0.6), Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612, Training Time: 89.25238418579102
Adam (lr=0.001), Validation Accuracy: 0.5000900030136108, Test Accuracy: 0.49869999289512634, Training Time: 106.54631996154785
Adam (lr=0.003), Validation Accuracy: 0.471647173166275, Test Accuracy: 0.4731000065803528, Training Time: 107.21505951881409
Adam (lr=0.006), Validation Accuracy: 0.41728171706199646, Test Accuracy: 0.39149999618530273, Training Time: 106.76777815818787
Adam (lr=0.01), Validation Accuracy: 0.392619252204895, Test Accuracy: 0.3531999886035919, Training Time: 105.85737895965576
Adam (lr=0.03), Validation Accuracy: 0.2018001824617386, Test Accuracy: 0.10000000149011612, Training Time: 105.79345917701721
Adam (lr=0.06), Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612, Training Time: 105.20845293998718
Adam (lr=0.1), Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612, Training Time: 106.4516031742096
Adam (lr=0.3), Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612, Training Time: 106.20232963562012
Adam (lr=0.6), Validation Accuracy: 0.10009001195430756, Test Accuracy: 0.10000000149011612, Training Time: 105.8603503704071

Experiment 2.7: Analize the optimal splitting of data into training, validation, and testing datasets.

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models, optimizers
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

def create_cnn_model():
    model = models.Sequential([
        layers.Flatten(input_shape=(32, 32, 3)),
        layers.Dense(256, activation='relu'),
        layers.Dense(128, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(10, activation='softmax')
    ])
    return model

def compile_and_train_model(train_images, train_labels, val_images, val_labels, test_images, test_labels, optimizer, epochs=20, batch_size=64):
    model = create_cnn_model()
    model.compile(optimizer=optimizer,
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    history = model.fit(train_images, train_labels, epochs=epochs, batch_size=batch_size,
                        validation_data=(val_images, val_labels), verbose=0)

    val_acc = max(history.history['val_accuracy'])
    test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)

    return val_acc, test_acc

split_ratios = [
    (0.70, 0.15, 0.15),
    (0.80, 0.10, 0.10),
    (0.90, 0.05, 0.05),
    (0.60, 0.20, 0.20),
    (0.50, 0.25, 0.25),
    (0.40, 0.30, 0.30),
    (0.75, 0.15, 0.10),
    (0.85, 0.10, 0.05),
    (0.65, 0.20, 0.15)
]

split_names = []
val_accuracies = []
test_accuracies = []

for train_ratio, val_ratio, test_ratio in split_ratios:
    total_size = len(train_images) + len(test_images)
    train_size = int(total_size * train_ratio)
    val_size = int(total_size * val_ratio)
    test_size = total_size - train_size - val_size

    combined_images = np.concatenate((train_images, test_images))
    combined_labels = np.concatenate((train_labels, test_labels))

    train_images_split, temp_images, train_labels_split, temp_labels = train_test_split(
        combined_images, combined_labels, train_size=train_size, random_state=42, stratify=combined_labels)
    val_images_split, test_images_split, val_labels_split, test_labels_split = train_test_split(
        temp_images, temp_labels, test_size=test_size, random_state=42, stratify=temp_labels)

    optimizer = optimizers.SGD(learning_rate=0.001, momentum=0.9)
    
    val_acc, test_acc = compile_and_train_model(train_images_split, train_labels_split, val_images_split, val_labels_split, test_images_split, test_labels_split,
                                                optimizer=optimizer, epochs=50, batch_size=64)
    
    split_names.append(f'{int(train_ratio*100)}/{int(val_ratio*100)}/{int(test_ratio*100)}')
    val_accuracies.append(val_acc)
    test_accuracies.append(test_acc)
    print(f'Split {int(train_ratio*100)}/{int(val_ratio*100)}/{int(test_ratio*100)}, Validation Accuracy: {val_acc:.4f}, Test Accuracy: {test_acc:.4f}')

plt.figure(figsize=(10, 6))

plt.plot(split_names, val_accuracies, marker='o', linestyle='-', color='b', label='Validation Accuracy')
plt.plot(split_names, test_accuracies, marker='o', linestyle='-', color='g', label='Test Accuracy')

plt.title('Validation and Test Accuracy vs. Data Splits')
plt.xlabel('Data Splits (Train/Val/Test)')
plt.ylabel('Accuracy')
plt.xticks(rotation=45)
plt.legend()
plt.grid(True)
plt.tight_layout()

plt.show()

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Split 70/15/15, Validation Accuracy: 0.5310, Test Accuracy: 0.5232

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Split 80/10/10, Validation Accuracy: 0.5380, Test Accuracy: 0.5185

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Split 90/5/5, Validation Accuracy: 0.5433, Test Accuracy: 0.5303

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Split 60/20/20, Validation Accuracy: 0.5204, Test Accuracy: 0.5015

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Split 50/25/25, Validation Accuracy: 0.5134, Test Accuracy: 0.4991

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Split 40/30/30, Validation Accuracy: 0.5066, Test Accuracy: 0.4849

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Split 75/15/10, Validation Accuracy: 0.5328, Test Accuracy: 0.5283

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Split 85/10/5, Validation Accuracy: 0.5408, Test Accuracy: 0.5347

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\reshaping\flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)

Split 65/20/15, Validation Accuracy: 0.5247, Test Accuracy: 0.5190

Experiment 2.8: Understand the role of Regularization techniques and Implement L1, L2, Dropout, Early stopping and batch normalizationtechniques and study the effect of these techniques on models’ performance

import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets, layers, models, optimizers, regularizers, callbacks
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_labels = train_labels.flatten()
test_labels = test_labels.flatten()

train_images, test_images = train_images / 255.0, test_images / 255.0

def create_cnn_model(regularization=None, dropout_rate=0.0, batch_norm=False):
    model = models.Sequential()
    
    model.add(layers.Conv2D(256, (3, 3), activation='relu', padding='same', input_shape=(32, 32, 3),
                            kernel_regularizer=regularization))
    if batch_norm:
        model.add(layers.BatchNormalization())
    model.add(layers.MaxPooling2D((2, 2)))
    
    model.add(layers.Conv2D(256, (3, 3), activation='relu', padding='same', kernel_regularizer=regularization))
    if batch_norm:
        model.add(layers.BatchNormalization())
    model.add(layers.MaxPooling2D((2, 2)))
    
    model.add(layers.Conv2D(128, (3, 3), activation='relu', padding='same', kernel_regularizer=regularization))
    if batch_norm:
        model.add(layers.BatchNormalization())
    
    model.add(layers.Flatten())
    
    model.add(layers.Dense(128, activation='relu', kernel_regularizer=regularization))
    if batch_norm:
        model.add(layers.BatchNormalization())
    model.add(layers.Dropout(dropout_rate))
    
    model.add(layers.Dense(128, activation='relu', kernel_regularizer=regularization))
    if batch_norm:
        model.add(layers.BatchNormalization())
    model.add(layers.Dropout(dropout_rate))
    
    model.add(layers.Dense(64, activation='relu', kernel_regularizer=regularization))
    if batch_norm:
        model.add(layers.BatchNormalization())
    model.add(layers.Dropout(dropout_rate))
    
    model.add(layers.Dense(64, activation='relu', kernel_regularizer=regularization))
    if batch_norm:
        model.add(layers.BatchNormalization())
    model.add(layers.Dropout(dropout_rate))
    
    model.add(layers.Dense(10, activation='softmax'))
    
    return model

def compile_and_train_model(model, optimizer, epochs=20, batch_size=64, use_early_stopping=False):
    model.compile(optimizer=optimizer,
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    if use_early_stopping:
        early_stopping = callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)
        callback_list = [early_stopping]
    else:
        callback_list = []
    
    history = model.fit(train_images, train_labels, epochs=epochs, batch_size=batch_size,
                        validation_data=(val_images, val_labels), callbacks=callback_list, verbose=0)

    val_acc = max(history.history['val_accuracy'])
    train_acc = max(history.history['accuracy'])
    test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)
    
    return train_acc, val_acc, test_acc, history.history['accuracy']

techniques = [ 'L1', 'L2', 'Dropout', 'BatchNorm', 'EarlyStopping']
train_accuracies = []
val_accuracies = []
test_accuracies = []

for technique in techniques:
    print(f'Running experiment with {technique}')
    
    if technique == 'L1':
        model = create_cnn_model(regularization=regularizers.l1(0.001))
        use_early_stopping = False
    elif technique == 'L2':
        model = create_cnn_model(regularization=regularizers.l2(0.001))
        use_early_stopping = False
    elif technique == 'Dropout':
        model = create_cnn_model(dropout_rate=0.5)
        use_early_stopping = False
    elif technique == 'BatchNorm':
        model = create_cnn_model(batch_norm=True)
        use_early_stopping = False
    elif technique == 'EarlyStopping':
        model = create_cnn_model()
        use_early_stopping = True
    
    optimizer = optimizers.SGD(learning_rate=0.001, momentum=0.9)
    
    train_acc, val_acc, test_acc, training_acc_history = compile_and_train_model(model, optimizer, epochs=15, batch_size=64, use_early_stopping=use_early_stopping)
    
    train_accuracies.append(train_acc)
    val_accuracies.append(val_acc)
    test_accuracies.append(test_acc)
    print(f'{technique} - Train Accuracy: {train_acc:.4f}, Validation Accuracy: {val_acc:.4f}, Test Accuracy: {test_acc:.4f}')

plt.figure(figsize=(10, 6))

plt.plot(techniques, train_accuracies, marker='o', linestyle='-', color='r', label='Training Accuracy')
plt.plot(techniques, val_accuracies, marker='o', linestyle='-', color='b', label='Validation Accuracy')
plt.plot(techniques, test_accuracies, marker='o', linestyle='-', color='g', label='Test Accuracy')

plt.title('Training, Validation, and Test Accuracy vs. Regularization Techniques')
plt.xlabel('Regularization Techniques')
plt.ylabel('Accuracy')
plt.xticks(rotation=45)
plt.legend()
plt.grid(True)
plt.tight_layout()

plt.show()

Running experiment with L1

C:\Users\mmac259\AppData\Roaming\Python\Python310\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

L1 - Train Accuracy: 0.3619, Validation Accuracy: 0.3703, Test Accuracy: 0.3768
Running experiment with L2
L2 - Train Accuracy: 0.6869, Validation Accuracy: 0.6781, Test Accuracy: 0.6782
Running experiment with Dropout
Dropout - Train Accuracy: 0.2798, Validation Accuracy: 0.3159, Test Accuracy: 0.3185
Running experiment with BatchNorm
BatchNorm - Train Accuracy: 0.9595, Validation Accuracy: 0.7163, Test Accuracy: 0.7139
Running experiment with EarlyStopping
EarlyStopping - Train Accuracy: 0.7208, Validation Accuracy: 0.6922, Test Accuracy: 0.6897